# -*- coding: utf-8 -*-
"""train_xgbost.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/19BMRHfEqZRhjUEF4uG_ogw8-v6mYkS-G
"""

from google.colab import drive
drive.mount('/content/drive', force_remount=True)

import sys
sys.path.append('/content/drive/My Drive/Stance Detection')

# !pip install transformers
import torch
if torch.cuda.is_available():
  device = torch.device("cuda")
else:
  device = torch.device("cpu")

print(device)

from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())

import nltk, string
from nltk.corpus import stopwords
from nltk import download
from nltk.util import ngrams
from nltk.corpus import wordnet
from nltk.sentiment.vader import SentimentIntensityAnalyzer

nltk.download('averaged_perceptron_tagger')
nltk.download('vader_lexicon')
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
download('stopwords').

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.decomposition import NMF as NonNegativeMatrixFactorization
import gensim.downloader as api

import pandas as pd
import tqdm

from nn_model import ConvEmbed
from nn_model import Classifier

from generate_embeddings import DatasetIterator
from torch.utils.data import DataLoader

PATH_TRAIN_STANCES = '/content/drive/My Drive/Stance Detection/dataset/train_stances.csv'
PATH_TRAIN_BODIES = '/content/drive/My Drive/Stance Detection/dataset/train_bodies.csv'

PATH_TEST_STANCES = '/content/drive/My Drive/Stance Detection/dataset/test_stances.csv'
PATH_TEST_BODIES = '/content/drive/My Drive/Stance Detection/dataset/test_bodies.csv'

class TfIdf:

  def __init__(self):
    self.lemmer = nltk.stem.WordNetLemmatizer()
    self.remove_punct_dict = dict((ord(punct), None) for punct in string.punctuation)
    self.vectorizer = TfidfVectorizer(tokenizer=self.lem_normalize, stop_words='english', ngram_range=(1, 3))

  def get_wordnet_pos(self, word):
    tag = nltk.pos_tag([word])[0][1][0].upper()
    tag_dict = {"J": wordnet.ADJ,
                "N": wordnet.NOUN,
                "V": wordnet.VERB,
                "R": wordnet.ADV}

    return tag_dict.get(tag, wordnet.NOUN)

  def lem_tokens(self, tokens):
    return [self.lemmer.lemmatize(token, self.get_wordnet_pos(token)) for token in tokens]

  def lem_normalize(self, text):
    return self.lem_tokens(nltk.word_tokenize(text.lower().translate(self.remove_punct_dict)))

  def compute(self, corpus):
    tfidf = self.vectorizer.fit_transform(corpus)
    return ((tfidf*tfidf.T).A)[0, 1]

  def process(self, df1, df2, df, path):
    tfidf_score = []

    data = pd.DataFrame()
    data['Headline ID'] = df['Headline ID']

    print('Computing tfidf_score')
    tfidf_score = [self.compute([headline, body]) for headline, body in tqdm.tqdm(zip(df['Headline'], df['articleBody']), position=0, leave=True)]

    data['tfidf_score'] = tfidf_score
    data.to_csv(path, index=False)

class LSA:
  def __init__(self):
    self.tfidf = TfIdf()
    self.lsa = TruncatedSVD(n_components=300, algorithm='randomized')

  def svd(self, corpus):
    tfidf = self.tfidf.vectorizer.fit_transform(corpus)
    return self.lsa.fit_transform(tfidf)
  
  def process(self, df1, df2, df, path):
    corpus = df1['Headline'].to_list() + df2['articleBody'].to_list()

    data = pd.DataFrame()
    data['Headline ID'] = df['Headline ID']

    lsa_cos = []
    lsa = self.svd(corpus)
    print('Computing svd')
    lsa_cos = [cosine_similarity(lsa[i].reshape(1, -1), lsa[j].reshape(1, -1))[0][0] for i, j in tqdm.tqdm(zip(df['Headline ID'], df['articleBody ID']))]

    data['lsa_cos'] = lsa_cos
    del lsa
    gc.collect()

    data.to_csv(path, index=False)

class LDA:
  def __init__(self):
    self.lemmer = nltk.stem.WordNetLemmatizer()
    self.remove_punct_dict = dict((ord(punct), None) for punct in string.punctuation)
    self.vectorizer = CountVectorizer(analyzer='word', tokenizer=self.lem_normalize, ngram_range=(1, 2), max_df=0.9, stop_words='english', lowercase=True, token_pattern='[a-zA-Z0-9]{3,}')
    self.lda_model = LatentDirichletAllocation(n_components=25, max_iter=50, n_jobs = -1, learning_method='online', verbose=1)
  def get_wordnet_pos(self, word):
    tag = nltk.pos_tag([word])[0][1][0].upper()
    tag_dict = {"J": wordnet.ADJ,
                "N": wordnet.NOUN,
                "V": wordnet.VERB,
                "R": wordnet.ADV}

    return tag_dict.get(tag, wordnet.NOUN)

  def lem_tokens(self, tokens):
    return [self.lemmer.lemmatize(token, self.get_wordnet_pos(token)) for token in tokens]

  def lem_normalize(self, text):
    tokenized = self.lem_tokens(nltk.word_tokenize(text.lower().translate(self.remove_punct_dict)))
    is_noun = lambda pos: pos[:2] == 'NN'
    nouns = [word for (word, pos) in nltk.pos_tag(tokenized) if is_noun(pos)]
    return nouns

  def compute(self, corpus):
    data_vectorized = self.vectorizer.fit_transform(corpus)
    lda_output = self.lda_model.fit_transform(data_vectorized)
    return lda_output
  
  def process(self, df1, df2, df, path):
    corpus = df1['Headline'].to_list() + df2['articleBody'].to_list()

    lda_cos = []
    data = pd.DataFrame()
    data['Headline ID'] = df['Headline ID']

    lda = self.compute(corpus)
    print('Computing lda')
    lda_cos = [cosine_similarity(lda[i].reshape(1, -1), lda[j].reshape(1, -1))[0][0] for i, j in tqdm.tqdm(zip(df['Headline ID'], df['articleBody ID']), position=0, leave=True)]

    data['lda_cos'] = lda_cos
    del lda
    gc.collect()

    data.to_csv(path, index=False)

class NMF:
  def __init__(self):
    self.nmf = NonNegativeMatrixFactorization(n_components=300, init='random', verbose=1, max_iter=50)
    self.tfidf = TfIdf()
  
  def compute(self, corpus):
    tfidf = self.tfidf.vectorizer.fit_transform(corpus)
    return self.nmf.fit_transform(tfidf)
  
  def process(self, df1, df2, df, path):
    corpus = df1['Headline'].to_list() + df2['articleBody'].to_list()

    nmf_cos = []
    data = pd.DataFrame()
    data['Headline ID'] = df['Headline ID']

    nmf = self.compute(corpus)
    print('Computing nmf')
    nmf_cos = [cosine_similarity(nmf[i].reshape(1, -1), nmf[j].reshape(1, -1))[0][0] for i, j in tqdm.tqdm(zip(df['Headline ID'], df['articleBody ID']), position=0, leave=True)]

    data['nmf_cos'] = nmf_cos
    del nmf
    gc.collect()

    data.to_csv(path, index=False)

class WMD:

  def __init__(self):
    self.stop_words = stopwords.words('english')
    self.model = api.load('word2vec-google-news-300')
    self.model.init_sims(replace=True)  # Normalizes the vectors in the word2vec class.

  def preprocess(self, sentence):
    return [w for w in sentence.lower().split() if w not in self.stop_words]

  def distance(self, headline, body):
    headline = self.preprocess(headline)
    body = self.preprocess(body)
    return self.model.wmdistance(headline, body)
  
  def process(self, df1, df2, df, path):
    wmd_score = []
    data = pd.DataFrame()
    data['Headline ID'] = df['Headline ID']
    print('Computing wmd_score')
    wmd_score = [(self.distance(headline, body)) for headline, body in tqdm.tqdm(zip(df['Headline'], df['articleBody']), position=0, leave=True)]
    data['wmd_score'] = wmd_score
    data.to_csv(path, index=False)

class Sentiment:

  def __init__(self):
    self.sia = SentimentIntensityAnalyzer()
    self.stemmer = nltk.stem.porter.PorterStemmer()
    self.remove_punctuation_map = dict((ord(char), None) for char in string.punctuation)

  def compute(self, headline, body, senti):
    headline_sentiment = self.sia.polarity_scores(headline)
    body_sentiment = self.sia.polarity_scores(body)
    senti['head_pos'].append(headline_sentiment['pos'])
    senti['body_pos'].append(body_sentiment['pos'])
    senti['head_neu'].append(headline_sentiment['neu'])
    senti['body_neu'].append(body_sentiment['neu'])
    senti['head_neg'].append(headline_sentiment['neg'])
    senti['body_neg'].append(body_sentiment['neg'])

  def process(self, df1, df2, df, path):
    senti_score = dict({
        'Headline ID': df['Headline ID'].to_list(),
        'head_pos': [],
        'body_pos': [],
        'head_neu': [],
        'body_neu': [],
        'head_neg': [],
        'body_neg': []
    })

    progress = tqdm.tqdm(total=df.shape[0], position=0, leave=True)
    print('Computing senti_score')
    [self.compute(headline, body, senti_score) for headline, body in tqdm.tqdm(zip(df['Headline'], df['articleBody']), position=0, leave=True)]

    data = pd.DataFrame(senti_score)
    data.to_csv(path, index=False)

class Stance:
  def __init__(self, neg_idx, device='cpu'):
    self.pathl = '/content/drive/My Drive/Stance Detection/logistic{}.pt'
    self.paths = '/content/drive/My Drive/Stance Detection/conv_model{}.pt'

    self.neg_idx = neg_idx
    self.device = device

    self.siamese = ConvEmbed()
    self.siamese.load_state_dict(torch.load(self.paths.format(self.neg_idx), map_location=self.device))
    self.siamese.to(self.device)
    self.siamese.eval()

    self.model = Classifier()
    self.model.load_state_dict(torch.load(self.pathl.format(neg_idx), map_location=self.device))
    self.model.to(self.device)
    self.model.eval()

  def scorer(self, headEmbedding, bodyEmbedding):
    with torch.no_grad():
      embed1 = self.siamese.forward(headEmbedding)
      embed2 = self.siamese.forward(bodyEmbedding)

      return self.model(embed1, embed2)

class CountVectorizer:

  def __init__(self):
    self.lemmer = nltk.stem.WordNetLemmatizer()
    self.remove_punct_dict = dict((ord(punct), None) for punct in string.punctuation)

  def get_wordnet_pos(self, word):
    tag = nltk.pos_tag([word])[0][1][0].upper()
    tag_dict = {"J": wordnet.ADJ,
                "N": wordnet.NOUN,
                "V": wordnet.VERB,
                "R": wordnet.ADV}

    return tag_dict.get(tag, wordnet.NOUN)

  def lem_tokens(self, tokens):
    return [self.lemmer.lemmatize(token, self.get_wordnet_pos(token)) for token in tokens]

  def lem_normalize(self, text):
    tokenized = self.lem_tokens(nltk.word_tokenize(text.lower().translate(self.remove_punct_dict)))
    return tokenized

  def try_divide(self, x, y, val=0.0):
    if y != 0.0:
        val = float(x) / y
    return val

  def extract_ngrams(self, data, num):
    n_grams = ngrams(self.lem_normalize(data), num)
    return [' '.join(grams) for grams in n_grams]

  def process(self, df1, df2, data, path):
    df = pd.DataFrame()
    df['Headline ID'] = data['Headline ID']
    print('Computing count features')
    grams = ["unigram", "bigram", "trigram"]
    feat_names = ["Headline", "articleBody"]
    n_grams = {'Headline': {'unigram': [], 'bigram': [], 'trigram': []}, 'articleBody': {'unigram': [], 'bigram': [], 'trigram': []}}

    for feat in feat_names:
      for i, gram in zip(range(1, 4), grams):
        n_grams[feat][gram] = [self.extract_ngrams(row, i) for row in tqdm.tqdm(data[feat], position=0, leave=True)]

    for feat_name in feat_names:
      for gram in grams:
        df["count_of_%s_%s" % (feat_name, gram)] = [len(x) for x in n_grams[feat_name][gram]]
        df["count_of_unique_%s_%s" % (feat_name, gram)] = [len(set(x)) for x in n_grams[feat_name][gram]]
        df["ratio_of_unique_%s_%s" % (feat_name, gram)] = [self.try_divide(x, y) for x, y in zip(df["count_of_unique_%s_%s"%(feat_name,gram)], df["count_of_%s_%s"%(feat_name,gram)])]

    # overlapping n-grams count
    for gram in grams:
      cnt = []
      df["count_of_Headline_%s_in_articleBody" % gram] = [sum([1. for w in x if w in set(y)]) for x, y in zip(n_grams['Headline'][gram], n_grams['articleBody'][gram])]
      df["ratio_of_Headline_%s_in_articleBody" % gram] = [self.try_divide(x, y) for x, y in zip(df["count_of_Headline_%s_in_articleBody" % gram], df["count_of_Headline_%s" % gram])]
        
    # binary refuting features
    _refuting_words = [
        'fake',
        'fraud',
        'hoax',
        'false',
        'deny', 'denies',
        # 'refute',
        'not',
        'despite',
        'nope',
        'doubt', 'doubts',
        'bogus',
        'debunk',
        'pranks',
        'retract'
    ]

    for rf in _refuting_words:
      fname = '%s_exist' % rf
      df[fname] = data['Headline'].map(lambda x: 1 if rf in x else 0)
    df.to_csv(path, index=False)

cv = CountVectorizer()
data = 'A class is a running run blueprint for the object.'

print("1-gram: ", cv.extract_ngrams(data, 1))
print("2-gram: ", cv.extract_ngrams(data, 2))
print("3-gram: ", cv.extract_ngrams(data, 3))
print("4-gram: ", cv.extract_ngrams(data, 4))

df1 = pd.DataFrame(pd.read_csv(PATH_TRAIN_STANCES))
df2 = pd.DataFrame(pd.read_csv(PATH_TRAIN_BODIES))
df2['articleBody ID'] = [i + df1.shape[0] for i in range(df2.shape[0])]
df = pd.merge(df1, df2, on='Body ID', how='inner')

_df1 = pd.DataFrame(pd.read_csv(PATH_TEST_STANCES))
_df2 = pd.DataFrame(pd.read_csv(PATH_TEST_BODIES))
_df2['articleBody ID'] = [i + _df1.shape[0] for i in range(_df2.shape[0])]
_df = pd.merge(_df1, _df2, on='Body ID', how='inner')

custom_path = '/content/drive/My Drive/Stance Detection/custom_data/{}.csv'

model = CountVectorizer()
model.process(_df1, _df2, _df, custom_path.format('count_test'))
model.process(df1, df2, df, custom_path.format('count'))

model = NMF()
model.process(df1, df2, df, custom_path.format('nmf_csv'))
model.process(_df1, _df2, _df, custom_path.format('nmf_test_csv'))

model = LDA()
model.process(_df1, _df2, _df, custom_path.format('lda_test_csv'))
model.process(df1, df2, df, custom_path.format('lda_csv'))

model = LSA()
model.process(df1, df2, df, custom_path.format('lsa_csv'))
model.process(_df1, _df2, _df, custom_path.format('lsa_test_csv'))

model = TfIdf()
model.process(df1, df2, df, custom_path.format('tfidf_csv'))
model.process(_df1, _df2, _df, custom_path.format('tfidf_test_csv'))

model = Sentiment()
model.process(_df1, _df2, _df, custom_path.format('sentiment_test_csv'))
model.process(df1, df2, df, custom_path.format('sentiment_csv'))

model = WMD()
model.process(_df1, _df2, _df, custom_path.format('wmd_test_csv'))
model.process(df1, df2, df, custom_path.format('wmd_csv'))

import pandas as pd
custom_path = '/content/drive/My Drive/Stance Detection/custom_data/{}.csv'
count_features = pd.DataFrame(pd.read_csv(custom_path.format('count_csv')))
count_features = pd.merge(df, count_features, on='Headline ID', how='inner')
count_features.head()

cv = CountVectorizer()
cv.count_features(df, custom_path.format('count_test_csv'))

import xgboost as xgb
import nltk, string
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np
import tqdm
from xgboost import plot_tree
from nltk.corpus import wordnet
nltk.download('averaged_perceptron_tagger')
from sklearn.ensemble import AdaBoostClassifier
nltk.download('wordnet')
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.metrics import f1_score
import tqdm
from math import sqrt
import gc
from functools import reduce

class XGBOOST:
  
  def __init__(self):
    self.path = '/content/drive/My Drive/Stance Detection/xgb_model.json'
    self.custom_path = '/content/drive/My Drive/Stance Detection/custom_data/{}.csv'
    self.xg_reg = xgb.XGBClassifier(objective ='multi:softmax')
    self.abc = AdaBoostClassifier(n_estimators=50, learning_rate=1)

  def map_stances(self, stance):
    if stance == 'unrelated':
      return 0
    if stance == 'disagree':
      return 1
    if stance == 'agree':
      return 2
    if stance == 'discuss':
      return 3

  def transform_data(self, PATH_STANCES, PATH_BODIES, train=True):
    df1 = pd.DataFrame(pd.read_csv(PATH_STANCES))
    if 'Headline ID' not in df1.columns:
      df1.insert(0, 'Headline ID', [i for i in range(df1.shape[0])])
      df1.to_csv(PATH_STANCES, index=False)
    df2 = pd.DataFrame(pd.read_csv(PATH_BODIES))
    df2['articleBody ID'] = [i + df1.shape[0] for i in range(df2.shape[0])]

    # wmd_score = []
    # sentiment = dict({
    #     'head_pos': [],
    #     'body_pos': [],
    #     'head_neu': [],
    #     'body_neu': [],
    #     'head_neg': [],
    #     'body_neg': []
    # })
    # tfidf = []
    # lsa_cos = []
    # output = []

    corpus = df1['Headline'].to_list() + df2['articleBody'].to_list()
    df = pd.merge(df1, df2, on='Body ID', how='inner')

    # self.cv.count_features(df, self.path.format('count_csv'))

    # nmf = self.nmf.compute(corpus)
    # progress = tqdm.tqdm(total=df.shape[0], position=0, leave=True)
    # print('Computing nmf')
    # for i, j in zip(df['Headline ID'], df['articleBody ID']):
    #   progress.update(1)
    #   nmf_cos.append(cosine_similarity(nmf[i].reshape(1, -1), nmf[j].reshape(1, -1))[0][0])
    # del nmf
    # gc.collect()

    # lsa = self.lsa.svd(corpus)
    # progress = tqdm.tqdm(total=df.shape[0], position=0, leave=True)
    # print('Computing svd')
    # for i, j in zip(df['Headline ID'], df['articleBody ID']):
    #   progress.update(1)
    #   lsa_cos.append(cosine_similarity(lsa[i].reshape(1, -1), lsa[j].reshape(1, -1))[0][0])
    # del lsa
    # gc.collect()

    # lda = self.lda.compute(corpus)
    # progress = tqdm.tqdm(total=df.shape[0], position=0, leave=True)
    # print('Computing lda')
    # for i, j in zip(df['Headline ID'], df['articleBody ID']):
    #   progress.update(1)
    #   lda_cos.append(cosine_similarity(lda[i].reshape(1, -1), lda[j].reshape(1, -1))[0][0])
    # del lda
    # gc.collect()

    # model = api.load('word2vec-google-news-300')
    # model.init_sims(replace=True)  # Normalizes the vectors in the word2vec class.

    # self.wmd = WMD(model)

    # progress = tqdm.tqdm(total=df.shape[0], position=0, leave=True)
    # print('Computing wmd_score, sentiment, tfidf')
    # for headline, body, stance in zip(df['Headline'], df['articleBody'], df['Stance']):
    #   progress.update(1)
    #   wmd_score.append(self.wmd.distance(headline, body))
    #   self.sentiment.compute(headline, body, sentiment)
    #   tfidf.append(self.tfidf.compute([headline, body]))
    
    # del model
    # gc.collect()

    # data = {'Headline ID': df['Headline ID'].to_list(), 'tfidf': tfidf, 'lsa_cos': lsa_cos, 'wmd': wmd_score}
    # data.update(sentiment)
    if train:
      count_features = pd.DataFrame(pd.read_csv(self.custom_path.format('count')))
      wmd_features = pd.DataFrame(pd.read_csv(self.custom_path.format('wmd_csv')))
      tfidf_features = pd.DataFrame(pd.read_csv(self.custom_path.format('tfidf_csv')))
      sentiment_features = pd.DataFrame(pd.read_csv(self.custom_path.format('sentiment_csv')))
      lsa_features = pd.DataFrame(pd.read_csv(self.custom_path.format('lsa_csv')))
      # lda_features = pd.DataFrame(pd.read_csv(self.custom_path.format('lda_csv')))
      nmf_features = pd.DataFrame(pd.read_csv(self.custom_path.format('nmf_csv')))
    else:
      count_features = pd.DataFrame(pd.read_csv(self.custom_path.format('count_test')))
      wmd_features = pd.DataFrame(pd.read_csv(self.custom_path.format('wmd_test_csv')))
      tfidf_features = pd.DataFrame(pd.read_csv(self.custom_path.format('tfidf_test_csv')))
      sentiment_features = pd.DataFrame(pd.read_csv(self.custom_path.format('sentiment_test_csv')))
      lsa_features = pd.DataFrame(pd.read_csv(self.custom_path.format('lsa_test_csv')))
      # lda_features = pd.DataFrame(pd.read_csv(self.custom_path.format('lda_test_csv')))
      nmf_features = pd.DataFrame(pd.read_csv(self.custom_path.format('nmf_test_csv')))

    dfs = [wmd_features, tfidf_features, sentiment_features, lsa_features, nmf_features, count_features]
    data = reduce(lambda left,right: pd.merge(left,right,on='Headline ID', how='inner'), dfs)

    data['output'] = [self.map_stances(out) for out in df['Stance']]

    df = pd.DataFrame(data)
    
    if train:
      df.to_csv(self.custom_path.format('xgb_csv'), index=False)
    else:
      df.to_csv(self.custom_path.format('xgb_test_csv'), index=False)

  def train_test(self, PATH_STANCES, PATH_BODIES, length=2533, train=True):
    if train:
      df = pd.read_csv(self.custom_path.format('xg_bert_train'))
      df = df.loc[:, ~df.columns.str.contains('^Unnamed')]
      df2 = pd.read_csv(self.custom_path.format('xg_bert_validate'))
      df2 = df2.loc[:, ~df2.columns.str.contains('^Unnamed')]
      count = dict()
      for stance in df['output']:
        if stance not in count:
          count[stance] = 0
        else:
          count[stance] += 1
      # wt = [2.7 if i == 1 else 1.2 if i == 2 else 1 for i in df['output']]
      wt = [sqrt(min(count.values()) / count[i]) for i in df['output']]
      self.xg_reg.fit(df.iloc[:,:-1], df.iloc[:, -1], verbose=True, early_stopping_rounds=10, eval_set=[(df2.iloc[:, :-1], df2.iloc[:, -1])], eval_metric='mlogloss', sample_weight=wt)
      # self.model = self.abc.fit(df.iloc[:,:-1], df.iloc[:, -1])
      plot_confusion_matrix(self.xg_reg, df2.iloc[:, :-1], df2.iloc[:, -1], values_format='d', display_labels=["unrelated", "disagree", "agree", "discuss"])
      plot_confusion_matrix(self.xg_reg, df2.iloc[:, :-1], df2.iloc[:, -1], display_labels=["unrelated", "disagree", "agree", "discuss"], normalize='true')
      xgb.plot_importance(self.xg_reg)
      plt.show()
      preds = self.xg_reg.predict(df.iloc[:,:-1])
      print(accuracy_score(df.iloc[:, -1], preds))
      print(f1_score(preds, df.iloc[:, -1], average='macro'))
      # preds = self.model.predict(df.iloc[:,:-1])
      # print(accuracy_score(df.iloc[:, -1], preds))
      # print(f1_score(preds, df.iloc[:, -1], average='macro'))
    else:
      df = pd.read_csv(self.custom_path.format('xg_bert_test'))
      head_id = df['Unnamed: 0']
      print(df.columns)
      df = df.loc[:, ~df.columns.str.contains('^Unnamed')]
      preds = self.xg_reg.predict(df.iloc[:,:-1])
      print(accuracy_score(df.iloc[:, -1], preds))
      print(f1_score(preds, df.iloc[:, -1], average='macro'))
      plot_confusion_matrix(self.xg_reg, df.iloc[:,:-1], df.iloc[:, -1], values_format='d', display_labels=["unrelated", "disagree", "agree", "discuss"])
      plot_confusion_matrix(self.xg_reg, df.iloc[:,:-1], df.iloc[:, -1], display_labels=["unrelated", "disagree", "agree", "discuss"], normalize='true')
      xgb.plot_importance(self.xg_reg)
      plt.show()
      wdf = pd.DataFrame()
      wdf['Headline ID'] = head_id
      wdf['Results'] = preds
      wdf.to_csv(self.custom_path.format('results'))
      # preds = self.model.predict(df.iloc[:,:-1])
      # print(accuracy_score(df.iloc[:, -1], preds))
      # print(f1_score(preds, df.iloc[:, -1], average='macro'))
      # plot_confusion_matrix(self.model, df.iloc[:,:-1], df.iloc[:, -1], values_format='d', display_labels=["unrelated", "disagree", "agree", "discuss"])
      # plot_confusion_matrix(self.model, df.iloc[:,:-1], df.iloc[:, -1], display_labels=["unrelated", "disagree", "agree", "discuss"], normalize='true')

    # plt.rcParams['figure.figsize'] = [50, 10]
    plot_tree(self.xg_reg, num_trees=4, rankdir='LR')
    plt.show()
  
  def bert_process(self, PATH_STANCES, PATH_BODIES, length=2533, train=True):
    data = pd.read_csv(self.custom_path.format('xgb_csv'))
    print(data.columns)
    data.drop(columns=['Headline ID'], inplace=True)
    count = dict()
    for stance in data['output']:
      if stance not in count:
        count[stance] = 0
      else:
        count[stance] += 1
    X, y = data.iloc[:,:-1], data.iloc[:,-1]
    X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.2)
    # wt = [sqrt(min(count.values()) / count[i]) for i in y_train]
    xg_reg = xgb.XGBClassifier(objective ='multi:softmax')
    xg_reg.fit(X_train, y_train, verbose=True, early_stopping_rounds=10, eval_set=[(X_validate, y_validate)], eval_metric='mlogloss')
    
    if train:
      df = pd.read_csv(self.custom_path.format('xgb_csv'))
      preds = xg_reg.predict(X)
    else:
      data = pd.read_csv(self.custom_path.format('xgb_test_csv'))
      df = pd.read_csv(self.custom_path.format('xgb_test_csv'))
      data.drop(columns=['Headline ID'], inplace=True)
      X, y = data.iloc[:,:-1], data.iloc[:,-1]
      preds = xg_reg.predict(X)

    plot_confusion_matrix(xg_reg, X, y, values_format='d', display_labels=["unrelated", "disagree", "agree", "discuss"])
    plt.show()
      
    df.insert(0, 'xg_prediction', preds)
    df.insert(0, 'disagree_agree',  pd.Series())
    df.insert(0, 'agree_discuss',  pd.Series())
    df.set_index('Headline ID', inplace=True)
    df2 = pd.DataFrame(columns=df.columns)
    df3 = pd.DataFrame(columns=df.columns)

    progress = tqdm.tqdm(total=length, position=0, leave=True)

    if train:
      train_data_size = 2250
    else:
      train_data_size = 2587

    d = DatasetIterator(PATH_STANCES, PATH_BODIES, False, 0, train_data_size)
    while True:
      dataloader = d.next(progress)
      if dataloader is None:
        break
      for body_id, headEmbedding, bodyEmbedding, stance, headline_id in dataloader:
        df2 = self.process(df, df2, headline_id, bodyEmbedding, headEmbedding)
    
    df2.drop(columns=['xg_prediction'], inplace=True)
    if train:
      df2.to_csv(self.custom_path.format('xg_bert_train'))
      d = DatasetIterator(PATH_STANCES, PATH_BODIES, False, train_data_size)
      while True:
        dataloader = d.next(progress)
        if dataloader is None:
          break
        for body_id, headEmbedding, bodyEmbedding, stance, headline_id in dataloader:
          df3 = self.process(df, df3, headline_id, bodyEmbedding, headEmbedding)
      df3.drop(columns=['xg_prediction'], inplace=True)
      df3.to_csv(self.custom_path.format('xg_bert_validate'))
    else:
      df2.to_csv(self.custom_path.format('xg_bert_test'))

  def process(self, df, df2, headline_id, bodyEmbedding, headEmbedding):
    bodyEmbedding = bodyEmbedding.squeeze(0)
    headEmbedding = headEmbedding.squeeze(0)
    for idx in range(len(headline_id)):
      id = headline_id[idx].item()
      df2.loc[id] = df.loc[id]
      if df.loc[id, 'xg_prediction'] == 0:
        continue
      neg_idx = 1
      while neg_idx < 3:
        headEmbed = headEmbedding[idx].unsqueeze(0).unsqueeze(0)
        bodyEmbed = bodyEmbedding.unsqueeze(0)
        bert_score = self.stance[neg_idx].scorer(headEmbed, bodyEmbed).item()
        if bert_score < 0.5:
          if neg_idx == 1:
            df2.loc[id, 'disagree_agree'] = bert_score
          elif neg_idx == 2:
            df2.loc[id, 'agree_discuss'] = bert_score
          break
        else:
          neg_idx += 1
          if neg_idx == 2:
            df2.loc[id, 'disagree_agree'] = bert_score
          elif neg_idx == 3:
            df2.loc[id, 'agree_discuss'] = bert_score
    return df2

xgb_classifier = XGBOOST()

xgb_classifier.transform_data(PATH_TRAIN_STANCES, PATH_TRAIN_BODIES)

xgb_classifier.transform_data(PATH_TEST_STANCES, PATH_TEST_BODIES, False)

xgb_classifier.bert_process(PATH_TRAIN_STANCES, PATH_TRAIN_BODIES)

xgb_classifier.bert_process(PATH_TEST_STANCES, PATH_TEST_BODIES, 2587, False)

xgb_classifier.train_test(PATH_TRAIN_STANCES, PATH_TRAIN_BODIES)

xgb_classifier.train_test(PATH_TEST_STANCES, PATH_TEST_BODIES, 2587, False)

xgb_classifier.train_test(PATH_TEST_STANCES, PATH_TEST_BODIES, False)
75ac0983320098b20ed8ceb97e466e5940b80eec
# -*- coding: utf-8 -*-
"""train_xgbost.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/19BMRHfEqZRhjUEF4uG_ogw8-v6mYkS-G
"""

from google.colab import drive
drive.mount('/content/drive', force_remount=True)

import sys
sys.path.append('/content/drive/My Drive/Stance Detection')

# !pip install transformers
import torch
if torch.cuda.is_available():
  device = torch.device("cuda")
else:
  device = torch.device("cpu")

print(device)

from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())

import nltk, string
from nltk.corpus import stopwords
from nltk import download
from nltk.util import ngrams
from nltk.corpus import wordnet
from nltk.sentiment.vader import SentimentIntensityAnalyzer

nltk.download('averaged_perceptron_tagger')
nltk.download('vader_lexicon')
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
download('stopwords').

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.decomposition import NMF as NonNegativeMatrixFactorization
import gensim.downloader as api

import pandas as pd
import tqdm

from nn_model import ConvEmbed
from nn_model import Classifier

from generate_embeddings import DatasetIterator
from torch.utils.data import DataLoader

PATH_TRAIN_STANCES = '/content/drive/My Drive/Stance Detection/dataset/train_stances.csv'
PATH_TRAIN_BODIES = '/content/drive/My Drive/Stance Detection/dataset/train_bodies.csv'

PATH_TEST_STANCES = '/content/drive/My Drive/Stance Detection/dataset/test_stances.csv'
PATH_TEST_BODIES = '/content/drive/My Drive/Stance Detection/dataset/test_bodies.csv'

class TfIdf:

  def __init__(self):
    self.lemmer = nltk.stem.WordNetLemmatizer()
    self.remove_punct_dict = dict((ord(punct), None) for punct in string.punctuation)
    self.vectorizer = TfidfVectorizer(tokenizer=self.lem_normalize, stop_words='english', ngram_range=(1, 3))

  def get_wordnet_pos(self, word):
    tag = nltk.pos_tag([word])[0][1][0].upper()
    tag_dict = {"J": wordnet.ADJ,
                "N": wordnet.NOUN,
                "V": wordnet.VERB,
                "R": wordnet.ADV}

    return tag_dict.get(tag, wordnet.NOUN)

  def lem_tokens(self, tokens):
    return [self.lemmer.lemmatize(token, self.get_wordnet_pos(token)) for token in tokens]

  def lem_normalize(self, text):
    return self.lem_tokens(nltk.word_tokenize(text.lower().translate(self.remove_punct_dict)))

  def compute(self, corpus):
    tfidf = self.vectorizer.fit_transform(corpus)
    return ((tfidf*tfidf.T).A)[0, 1]

  def process(self, df1, df2, df, path):
    tfidf_score = []

    data = pd.DataFrame()
    data['Headline ID'] = df['Headline ID']

    print('Computing tfidf_score')
    tfidf_score = [self.compute([headline, body]) for headline, body in tqdm.tqdm(zip(df['Headline'], df['articleBody']), position=0, leave=True)]

    data['tfidf_score'] = tfidf_score
    data.to_csv(path, index=False)

class LSA:
  def __init__(self):
    self.tfidf = TfIdf()
    self.lsa = TruncatedSVD(n_components=300, algorithm='randomized')

  def svd(self, corpus):
    tfidf = self.tfidf.vectorizer.fit_transform(corpus)
    return self.lsa.fit_transform(tfidf)
  
  def process(self, df1, df2, df, path):
    corpus = df1['Headline'].to_list() + df2['articleBody'].to_list()

    data = pd.DataFrame()
    data['Headline ID'] = df['Headline ID']

    lsa_cos = []
    lsa = self.svd(corpus)
    print('Computing svd')
    lsa_cos = [cosine_similarity(lsa[i].reshape(1, -1), lsa[j].reshape(1, -1))[0][0] for i, j in tqdm.tqdm(zip(df['Headline ID'], df['articleBody ID']))]

    data['lsa_cos'] = lsa_cos
    del lsa
    gc.collect()

    data.to_csv(path, index=False)

class LDA:
  def __init__(self):
    self.lemmer = nltk.stem.WordNetLemmatizer()
    self.remove_punct_dict = dict((ord(punct), None) for punct in string.punctuation)
    self.vectorizer = CountVectorizer(analyzer='word', tokenizer=self.lem_normalize, ngram_range=(1, 2), max_df=0.9, stop_words='english', lowercase=True, token_pattern='[a-zA-Z0-9]{3,}')
    self.lda_model = LatentDirichletAllocation(n_components=25, max_iter=50, n_jobs = -1, learning_method='online', verbose=1)
  def get_wordnet_pos(self, word):
    tag = nltk.pos_tag([word])[0][1][0].upper()
    tag_dict = {"J": wordnet.ADJ,
                "N": wordnet.NOUN,
                "V": wordnet.VERB,
                "R": wordnet.ADV}

    return tag_dict.get(tag, wordnet.NOUN)

  def lem_tokens(self, tokens):
    return [self.lemmer.lemmatize(token, self.get_wordnet_pos(token)) for token in tokens]

  def lem_normalize(self, text):
    tokenized = self.lem_tokens(nltk.word_tokenize(text.lower().translate(self.remove_punct_dict)))
    is_noun = lambda pos: pos[:2] == 'NN'
    nouns = [word for (word, pos) in nltk.pos_tag(tokenized) if is_noun(pos)]
    return nouns

  def compute(self, corpus):
    data_vectorized = self.vectorizer.fit_transform(corpus)
    lda_output = self.lda_model.fit_transform(data_vectorized)
    return lda_output
  
  def process(self, df1, df2, df, path):
    corpus = df1['Headline'].to_list() + df2['articleBody'].to_list()

    lda_cos = []
    data = pd.DataFrame()
    data['Headline ID'] = df['Headline ID']

    lda = self.compute(corpus)
    print('Computing lda')
    lda_cos = [cosine_similarity(lda[i].reshape(1, -1), lda[j].reshape(1, -1))[0][0] for i, j in tqdm.tqdm(zip(df['Headline ID'], df['articleBody ID']), position=0, leave=True)]

    data['lda_cos'] = lda_cos
    del lda
    gc.collect()

    data.to_csv(path, index=False)

class NMF:
  def __init__(self):
    self.nmf = NonNegativeMatrixFactorization(n_components=300, init='random', verbose=1, max_iter=50)
    self.tfidf = TfIdf()
  
  def compute(self, corpus):
    tfidf = self.tfidf.vectorizer.fit_transform(corpus)
    return self.nmf.fit_transform(tfidf)
  
  def process(self, df1, df2, df, path):
    corpus = df1['Headline'].to_list() + df2['articleBody'].to_list()

    nmf_cos = []
    data = pd.DataFrame()
    data['Headline ID'] = df['Headline ID']

    nmf = self.compute(corpus)
    print('Computing nmf')
    nmf_cos = [cosine_similarity(nmf[i].reshape(1, -1), nmf[j].reshape(1, -1))[0][0] for i, j in tqdm.tqdm(zip(df['Headline ID'], df['articleBody ID']), position=0, leave=True)]

    data['nmf_cos'] = nmf_cos
    del nmf
    gc.collect()

    data.to_csv(path, index=False)

class WMD:

  def __init__(self):
    self.stop_words = stopwords.words('english')
    self.model = api.load('word2vec-google-news-300')
    self.model.init_sims(replace=True)  # Normalizes the vectors in the word2vec class.

  def preprocess(self, sentence):
    return [w for w in sentence.lower().split() if w not in self.stop_words]

  def distance(self, headline, body):
    headline = self.preprocess(headline)
    body = self.preprocess(body)
    return self.model.wmdistance(headline, body)
  
  def process(self, df1, df2, df, path):
    wmd_score = []
    data = pd.DataFrame()
    data['Headline ID'] = df['Headline ID']
    print('Computing wmd_score')
    wmd_score = [(self.distance(headline, body)) for headline, body in tqdm.tqdm(zip(df['Headline'], df['articleBody']), position=0, leave=True)]
    data['wmd_score'] = wmd_score
    data.to_csv(path, index=False)

class Sentiment:

  def __init__(self):
    self.sia = SentimentIntensityAnalyzer()
    self.stemmer = nltk.stem.porter.PorterStemmer()
    self.remove_punctuation_map = dict((ord(char), None) for char in string.punctuation)

  def compute(self, headline, body, senti):
    headline_sentiment = self.sia.polarity_scores(headline)
    body_sentiment = self.sia.polarity_scores(body)
    senti['head_pos'].append(headline_sentiment['pos'])
    senti['body_pos'].append(body_sentiment['pos'])
    senti['head_neu'].append(headline_sentiment['neu'])
    senti['body_neu'].append(body_sentiment['neu'])
    senti['head_neg'].append(headline_sentiment['neg'])
    senti['body_neg'].append(body_sentiment['neg'])

  def process(self, df1, df2, df, path):
    senti_score = dict({
        'Headline ID': df['Headline ID'].to_list(),
        'head_pos': [],
        'body_pos': [],
        'head_neu': [],
        'body_neu': [],
        'head_neg': [],
        'body_neg': []
    })

    progress = tqdm.tqdm(total=df.shape[0], position=0, leave=True)
    print('Computing senti_score')
    [self.compute(headline, body, senti_score) for headline, body in tqdm.tqdm(zip(df['Headline'], df['articleBody']), position=0, leave=True)]

    data = pd.DataFrame(senti_score)
    data.to_csv(path, index=False)

class Stance:
  def __init__(self, neg_idx, device='cpu'):
    self.pathl = '/content/drive/My Drive/Stance Detection/logistic{}.pt'
    self.paths = '/content/drive/My Drive/Stance Detection/conv_model{}.pt'

    self.neg_idx = neg_idx
    self.device = device

    self.siamese = ConvEmbed()
    self.siamese.load_state_dict(torch.load(self.paths.format(self.neg_idx), map_location=self.device))
    self.siamese.to(self.device)
    self.siamese.eval()

    self.model = Classifier()
    self.model.load_state_dict(torch.load(self.pathl.format(neg_idx), map_location=self.device))
    self.model.to(self.device)
    self.model.eval()

  def scorer(self, headEmbedding, bodyEmbedding):
    with torch.no_grad():
      embed1 = self.siamese.forward(headEmbedding)
      embed2 = self.siamese.forward(bodyEmbedding)

      return self.model(embed1, embed2)

class CountVectorizer:

  def __init__(self):
    self.lemmer = nltk.stem.WordNetLemmatizer()
    self.remove_punct_dict = dict((ord(punct), None) for punct in string.punctuation)

  def get_wordnet_pos(self, word):
    tag = nltk.pos_tag([word])[0][1][0].upper()
    tag_dict = {"J": wordnet.ADJ,
                "N": wordnet.NOUN,
                "V": wordnet.VERB,
                "R": wordnet.ADV}

    return tag_dict.get(tag, wordnet.NOUN)

  def lem_tokens(self, tokens):
    return [self.lemmer.lemmatize(token, self.get_wordnet_pos(token)) for token in tokens]

  def lem_normalize(self, text):
    tokenized = self.lem_tokens(nltk.word_tokenize(text.lower().translate(self.remove_punct_dict)))
    return tokenized

  def try_divide(self, x, y, val=0.0):
    if y != 0.0:
        val = float(x) / y
    return val

  def extract_ngrams(self, data, num):
    n_grams = ngrams(self.lem_normalize(data), num)
    return [' '.join(grams) for grams in n_grams]

  def process(self, df1, df2, data, path):
    df = pd.DataFrame()
    df['Headline ID'] = data['Headline ID']
    print('Computing count features')
    grams = ["unigram", "bigram", "trigram"]
    feat_names = ["Headline", "articleBody"]
    n_grams = {'Headline': {'unigram': [], 'bigram': [], 'trigram': []}, 'articleBody': {'unigram': [], 'bigram': [], 'trigram': []}}

    for feat in feat_names:
      for i, gram in zip(range(1, 4), grams):
        n_grams[feat][gram] = [self.extract_ngrams(row, i) for row in tqdm.tqdm(data[feat], position=0, leave=True)]

    for feat_name in feat_names:
      for gram in grams:
        df["count_of_%s_%s" % (feat_name, gram)] = [len(x) for x in n_grams[feat_name][gram]]
        df["count_of_unique_%s_%s" % (feat_name, gram)] = [len(set(x)) for x in n_grams[feat_name][gram]]
        df["ratio_of_unique_%s_%s" % (feat_name, gram)] = [self.try_divide(x, y) for x, y in zip(df["count_of_unique_%s_%s"%(feat_name,gram)], df["count_of_%s_%s"%(feat_name,gram)])]

    # overlapping n-grams count
    for gram in grams:
      cnt = []
      df["count_of_Headline_%s_in_articleBody" % gram] = [sum([1. for w in x if w in set(y)]) for x, y in zip(n_grams['Headline'][gram], n_grams['articleBody'][gram])]
      df["ratio_of_Headline_%s_in_articleBody" % gram] = [self.try_divide(x, y) for x, y in zip(df["count_of_Headline_%s_in_articleBody" % gram], df["count_of_Headline_%s" % gram])]
        
    # binary refuting features
    _refuting_words = [
        'fake',
        'fraud',
        'hoax',
        'false',
        'deny', 'denies',
        # 'refute',
        'not',
        'despite',
        'nope',
        'doubt', 'doubts',
        'bogus',
        'debunk',
        'pranks',
        'retract'
    ]

    for rf in _refuting_words:
      fname = '%s_exist' % rf
      df[fname] = data['Headline'].map(lambda x: 1 if rf in x else 0)
    df.to_csv(path, index=False)

cv = CountVectorizer()
data = 'A class is a running run blueprint for the object.'

print("1-gram: ", cv.extract_ngrams(data, 1))
print("2-gram: ", cv.extract_ngrams(data, 2))
print("3-gram: ", cv.extract_ngrams(data, 3))
print("4-gram: ", cv.extract_ngrams(data, 4))

df1 = pd.DataFrame(pd.read_csv(PATH_TRAIN_STANCES))
df2 = pd.DataFrame(pd.read_csv(PATH_TRAIN_BODIES))
df2['articleBody ID'] = [i + df1.shape[0] for i in range(df2.shape[0])]
df = pd.merge(df1, df2, on='Body ID', how='inner')

_df1 = pd.DataFrame(pd.read_csv(PATH_TEST_STANCES))
_df2 = pd.DataFrame(pd.read_csv(PATH_TEST_BODIES))
_df2['articleBody ID'] = [i + _df1.shape[0] for i in range(_df2.shape[0])]
_df = pd.merge(_df1, _df2, on='Body ID', how='inner')

custom_path = '/content/drive/My Drive/Stance Detection/custom_data/{}.csv'

model = CountVectorizer()
model.process(_df1, _df2, _df, custom_path.format('count_test'))
model.process(df1, df2, df, custom_path.format('count'))

model = NMF()
model.process(df1, df2, df, custom_path.format('nmf_csv'))
model.process(_df1, _df2, _df, custom_path.format('nmf_test_csv'))

model = LDA()
model.process(_df1, _df2, _df, custom_path.format('lda_test_csv'))
model.process(df1, df2, df, custom_path.format('lda_csv'))

model = LSA()
model.process(df1, df2, df, custom_path.format('lsa_csv'))
model.process(_df1, _df2, _df, custom_path.format('lsa_test_csv'))

model = TfIdf()
model.process(df1, df2, df, custom_path.format('tfidf_csv'))
model.process(_df1, _df2, _df, custom_path.format('tfidf_test_csv'))

model = Sentiment()
model.process(_df1, _df2, _df, custom_path.format('sentiment_test_csv'))
model.process(df1, df2, df, custom_path.format('sentiment_csv'))

model = WMD()
model.process(_df1, _df2, _df, custom_path.format('wmd_test_csv'))
model.process(df1, df2, df, custom_path.format('wmd_csv'))

import pandas as pd
custom_path = '/content/drive/My Drive/Stance Detection/custom_data/{}.csv'
count_features = pd.DataFrame(pd.read_csv(custom_path.format('count_csv')))
count_features = pd.merge(df, count_features, on='Headline ID', how='inner')
count_features.head()

cv = CountVectorizer()
cv.count_features(df, custom_path.format('count_test_csv'))

import xgboost as xgb
import nltk, string
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np
import tqdm
from xgboost import plot_tree
from nltk.corpus import wordnet
nltk.download('averaged_perceptron_tagger')
from sklearn.ensemble import AdaBoostClassifier
nltk.download('wordnet')
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.metrics import f1_score
import tqdm
from math import sqrt
import gc
from functools import reduce

class XGBOOST:
  
  def __init__(self):
    self.path = '/content/drive/My Drive/Stance Detection/xgb_model.json'
    self.custom_path = '/content/drive/My Drive/Stance Detection/custom_data/{}.csv'
    self.xg_reg = xgb.XGBClassifier(objective ='multi:softmax')
    self.abc = AdaBoostClassifier(n_estimators=50, learning_rate=1)

  def map_stances(self, stance):
    if stance == 'unrelated':
      return 0
    if stance == 'disagree':
      return 1
    if stance == 'agree':
      return 2
    if stance == 'discuss':
      return 3

  def transform_data(self, PATH_STANCES, PATH_BODIES, train=True):
    df1 = pd.DataFrame(pd.read_csv(PATH_STANCES))
    if 'Headline ID' not in df1.columns:
      df1.insert(0, 'Headline ID', [i for i in range(df1.shape[0])])
      df1.to_csv(PATH_STANCES, index=False)
    df2 = pd.DataFrame(pd.read_csv(PATH_BODIES))
    df2['articleBody ID'] = [i + df1.shape[0] for i in range(df2.shape[0])]

    # wmd_score = []
    # sentiment = dict({
    #     'head_pos': [],
    #     'body_pos': [],
    #     'head_neu': [],
    #     'body_neu': [],
    #     'head_neg': [],
    #     'body_neg': []
    # })
    # tfidf = []
    # lsa_cos = []
    # output = []

    corpus = df1['Headline'].to_list() + df2['articleBody'].to_list()
    df = pd.merge(df1, df2, on='Body ID', how='inner')

    # self.cv.count_features(df, self.path.format('count_csv'))

    # nmf = self.nmf.compute(corpus)
    # progress = tqdm.tqdm(total=df.shape[0], position=0, leave=True)
    # print('Computing nmf')
    # for i, j in zip(df['Headline ID'], df['articleBody ID']):
    #   progress.update(1)
    #   nmf_cos.append(cosine_similarity(nmf[i].reshape(1, -1), nmf[j].reshape(1, -1))[0][0])
    # del nmf
    # gc.collect()

    # lsa = self.lsa.svd(corpus)
    # progress = tqdm.tqdm(total=df.shape[0], position=0, leave=True)
    # print('Computing svd')
    # for i, j in zip(df['Headline ID'], df['articleBody ID']):
    #   progress.update(1)
    #   lsa_cos.append(cosine_similarity(lsa[i].reshape(1, -1), lsa[j].reshape(1, -1))[0][0])
    # del lsa
    # gc.collect()

    # lda = self.lda.compute(corpus)
    # progress = tqdm.tqdm(total=df.shape[0], position=0, leave=True)
    # print('Computing lda')
    # for i, j in zip(df['Headline ID'], df['articleBody ID']):
    #   progress.update(1)
    #   lda_cos.append(cosine_similarity(lda[i].reshape(1, -1), lda[j].reshape(1, -1))[0][0])
    # del lda
    # gc.collect()

    # model = api.load('word2vec-google-news-300')
    # model.init_sims(replace=True)  # Normalizes the vectors in the word2vec class.

    # self.wmd = WMD(model)

    # progress = tqdm.tqdm(total=df.shape[0], position=0, leave=True)
    # print('Computing wmd_score, sentiment, tfidf')
    # for headline, body, stance in zip(df['Headline'], df['articleBody'], df['Stance']):
    #   progress.update(1)
    #   wmd_score.append(self.wmd.distance(headline, body))
    #   self.sentiment.compute(headline, body, sentiment)
    #   tfidf.append(self.tfidf.compute([headline, body]))
    
    # del model
    # gc.collect()

    # data = {'Headline ID': df['Headline ID'].to_list(), 'tfidf': tfidf, 'lsa_cos': lsa_cos, 'wmd': wmd_score}
    # data.update(sentiment)
    if train:
      count_features = pd.DataFrame(pd.read_csv(self.custom_path.format('count')))
      wmd_features = pd.DataFrame(pd.read_csv(self.custom_path.format('wmd_csv')))
      tfidf_features = pd.DataFrame(pd.read_csv(self.custom_path.format('tfidf_csv')))
      sentiment_features = pd.DataFrame(pd.read_csv(self.custom_path.format('sentiment_csv')))
      lsa_features = pd.DataFrame(pd.read_csv(self.custom_path.format('lsa_csv')))
      # lda_features = pd.DataFrame(pd.read_csv(self.custom_path.format('lda_csv')))
      nmf_features = pd.DataFrame(pd.read_csv(self.custom_path.format('nmf_csv')))
    else:
      count_features = pd.DataFrame(pd.read_csv(self.custom_path.format('count_test')))
      wmd_features = pd.DataFrame(pd.read_csv(self.custom_path.format('wmd_test_csv')))
      tfidf_features = pd.DataFrame(pd.read_csv(self.custom_path.format('tfidf_test_csv')))
      sentiment_features = pd.DataFrame(pd.read_csv(self.custom_path.format('sentiment_test_csv')))
      lsa_features = pd.DataFrame(pd.read_csv(self.custom_path.format('lsa_test_csv')))
      # lda_features = pd.DataFrame(pd.read_csv(self.custom_path.format('lda_test_csv')))
      nmf_features = pd.DataFrame(pd.read_csv(self.custom_path.format('nmf_test_csv')))

    dfs = [wmd_features, tfidf_features, sentiment_features, lsa_features, nmf_features, count_features]
    data = reduce(lambda left,right: pd.merge(left,right,on='Headline ID', how='inner'), dfs)

    data['output'] = [self.map_stances(out) for out in df['Stance']]

    df = pd.DataFrame(data)
    
    if train:
      df.to_csv(self.custom_path.format('xgb_csv'), index=False)
    else:
      df.to_csv(self.custom_path.format('xgb_test_csv'), index=False)

  def train_test(self, PATH_STANCES, PATH_BODIES, length=2533, train=True):
    if train:
      df = pd.read_csv(self.custom_path.format('xg_bert_train'))
      df = df.loc[:, ~df.columns.str.contains('^Unnamed')]
      df2 = pd.read_csv(self.custom_path.format('xg_bert_validate'))
      df2 = df2.loc[:, ~df2.columns.str.contains('^Unnamed')]
      count = dict()
      for stance in df['output']:
        if stance not in count:
          count[stance] = 0
        else:
          count[stance] += 1
      # wt = [2.7 if i == 1 else 1.2 if i == 2 else 1 for i in df['output']]
      wt = [sqrt(min(count.values()) / count[i]) for i in df['output']]
      self.xg_reg.fit(df.iloc[:,:-1], df.iloc[:, -1], verbose=True, early_stopping_rounds=10, eval_set=[(df2.iloc[:, :-1], df2.iloc[:, -1])], eval_metric='mlogloss', sample_weight=wt)
      # self.model = self.abc.fit(df.iloc[:,:-1], df.iloc[:, -1])
      plot_confusion_matrix(self.xg_reg, df2.iloc[:, :-1], df2.iloc[:, -1], values_format='d', display_labels=["unrelated", "disagree", "agree", "discuss"])
      plot_confusion_matrix(self.xg_reg, df2.iloc[:, :-1], df2.iloc[:, -1], display_labels=["unrelated", "disagree", "agree", "discuss"], normalize='true')
      xgb.plot_importance(self.xg_reg)
      plt.show()
      preds = self.xg_reg.predict(df.iloc[:,:-1])
      print(accuracy_score(df.iloc[:, -1], preds))
      print(f1_score(preds, df.iloc[:, -1], average='macro'))
      # preds = self.model.predict(df.iloc[:,:-1])
      # print(accuracy_score(df.iloc[:, -1], preds))
      # print(f1_score(preds, df.iloc[:, -1], average='macro'))
    else:
      df = pd.read_csv(self.custom_path.format('xg_bert_test'))
      head_id = df['Unnamed: 0']
      print(df.columns)
      df = df.loc[:, ~df.columns.str.contains('^Unnamed')]
      preds = self.xg_reg.predict(df.iloc[:,:-1])
      print(accuracy_score(df.iloc[:, -1], preds))
      print(f1_score(preds, df.iloc[:, -1], average='macro'))
      plot_confusion_matrix(self.xg_reg, df.iloc[:,:-1], df.iloc[:, -1], values_format='d', display_labels=["unrelated", "disagree", "agree", "discuss"])
      plot_confusion_matrix(self.xg_reg, df.iloc[:,:-1], df.iloc[:, -1], display_labels=["unrelated", "disagree", "agree", "discuss"], normalize='true')
      xgb.plot_importance(self.xg_reg)
      plt.show()
      wdf = pd.DataFrame()
      wdf['Headline ID'] = head_id
      wdf['Results'] = preds
      wdf.to_csv(self.custom_path.format('results'))
      # preds = self.model.predict(df.iloc[:,:-1])
      # print(accuracy_score(df.iloc[:, -1], preds))
      # print(f1_score(preds, df.iloc[:, -1], average='macro'))
      # plot_confusion_matrix(self.model, df.iloc[:,:-1], df.iloc[:, -1], values_format='d', display_labels=["unrelated", "disagree", "agree", "discuss"])
      # plot_confusion_matrix(self.model, df.iloc[:,:-1], df.iloc[:, -1], display_labels=["unrelated", "disagree", "agree", "discuss"], normalize='true')

    # plt.rcParams['figure.figsize'] = [50, 10]
    plot_tree(self.xg_reg, num_trees=4, rankdir='LR')
    plt.show()
  
  def bert_process(self, PATH_STANCES, PATH_BODIES, length=2533, train=True):
    data = pd.read_csv(self.custom_path.format('xgb_csv'))
    print(data.columns)
    data.drop(columns=['Headline ID'], inplace=True)
    count = dict()
    for stance in data['output']:
      if stance not in count:
        count[stance] = 0
      else:
        count[stance] += 1
    X, y = data.iloc[:,:-1], data.iloc[:,-1]
    X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.2)
    # wt = [sqrt(min(count.values()) / count[i]) for i in y_train]
    xg_reg = xgb.XGBClassifier(objective ='multi:softmax')
    xg_reg.fit(X_train, y_train, verbose=True, early_stopping_rounds=10, eval_set=[(X_validate, y_validate)], eval_metric='mlogloss')
    
    if train:
      df = pd.read_csv(self.custom_path.format('xgb_csv'))
      preds = xg_reg.predict(X)
    else:
      data = pd.read_csv(self.custom_path.format('xgb_test_csv'))
      df = pd.read_csv(self.custom_path.format('xgb_test_csv'))
      data.drop(columns=['Headline ID'], inplace=True)
      X, y = data.iloc[:,:-1], data.iloc[:,-1]
      preds = xg_reg.predict(X)

    plot_confusion_matrix(xg_reg, X, y, values_format='d', display_labels=["unrelated", "disagree", "agree", "discuss"])
    plt.show()
      
    df.insert(0, 'xg_prediction', preds)
    df.insert(0, 'disagree_agree',  pd.Series())
    df.insert(0, 'agree_discuss',  pd.Series())
    df.set_index('Headline ID', inplace=True)
    df2 = pd.DataFrame(columns=df.columns)
    df3 = pd.DataFrame(columns=df.columns)

    progress = tqdm.tqdm(total=length, position=0, leave=True)

    if train:
      train_data_size = 2250
    else:
      train_data_size = 2587

    d = DatasetIterator(PATH_STANCES, PATH_BODIES, False, 0, train_data_size)
    while True:
      dataloader = d.next(progress)
      if dataloader is None:
        break
      for body_id, headEmbedding, bodyEmbedding, stance, headline_id in dataloader:
        df2 = self.process(df, df2, headline_id, bodyEmbedding, headEmbedding)
    
    df2.drop(columns=['xg_prediction'], inplace=True)
    if train:
      df2.to_csv(self.custom_path.format('xg_bert_train'))
      d = DatasetIterator(PATH_STANCES, PATH_BODIES, False, train_data_size)
      while True:
        dataloader = d.next(progress)
        if dataloader is None:
          break
        for body_id, headEmbedding, bodyEmbedding, stance, headline_id in dataloader:
          df3 = self.process(df, df3, headline_id, bodyEmbedding, headEmbedding)
      df3.drop(columns=['xg_prediction'], inplace=True)
      df3.to_csv(self.custom_path.format('xg_bert_validate'))
    else:
      df2.to_csv(self.custom_path.format('xg_bert_test'))

  def process(self, df, df2, headline_id, bodyEmbedding, headEmbedding):
    bodyEmbedding = bodyEmbedding.squeeze(0)
    headEmbedding = headEmbedding.squeeze(0)
    for idx in range(len(headline_id)):
      id = headline_id[idx].item()
      df2.loc[id] = df.loc[id]
      if df.loc[id, 'xg_prediction'] == 0:
        continue
      neg_idx = 1
      while neg_idx < 3:
        headEmbed = headEmbedding[idx].unsqueeze(0).unsqueeze(0)
        bodyEmbed = bodyEmbedding.unsqueeze(0)
        bert_score = self.stance[neg_idx].scorer(headEmbed, bodyEmbed).item()
        if bert_score < 0.5:
          if neg_idx == 1:
            df2.loc[id, 'disagree_agree'] = bert_score
          elif neg_idx == 2:
            df2.loc[id, 'agree_discuss'] = bert_score
          break
        else:
          neg_idx += 1
          if neg_idx == 2:
            df2.loc[id, 'disagree_agree'] = bert_score
          elif neg_idx == 3:
            df2.loc[id, 'agree_discuss'] = bert_score
    return df2

xgb_classifier = XGBOOST()

xgb_classifier.transform_data(PATH_TRAIN_STANCES, PATH_TRAIN_BODIES)

xgb_classifier.transform_data(PATH_TEST_STANCES, PATH_TEST_BODIES, False)

xgb_classifier.bert_process(PATH_TRAIN_STANCES, PATH_TRAIN_BODIES)

xgb_classifier.bert_process(PATH_TEST_STANCES, PATH_TEST_BODIES, 2587, False)

xgb_classifier.train_test(PATH_TRAIN_STANCES, PATH_TRAIN_BODIES)

xgb_classifier.train_test(PATH_TEST_STANCES, PATH_TEST_BODIES, 2587, False)

xgb_classifier.train_test(PATH_TEST_STANCES, PATH_TEST_BODIES, False)