# Main source: https://www.kaggle.com/maksimeren/covid-19-literature-clustering

ROOT_PATH = 'C:\\Users\\amondejar\\Desktop\\SharedVM\\dataset'
QUANT_PAPERS = 50000
RANGE_INDEX = 44557
# RangeIndex samples:
# For 10000 papers, we have 8892 entries
# For 1000 papers, we have 898 entries
# For 500 papers, we have 454 entries
# For 200 papers, we have 180 entries
# More details? See line 188 and see what we have in variable 'count'

######################################################################

import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)
import glob
import json
import matplotlib.pyplot as plt

plt.style.use('ggplot')

# Let's load the metadata of the dateset. 'title' and 'journal' attributes may be useful later when we cluster the
# articles to see what kinds of articles cluster together.


root_path = ROOT_PATH
# root_path = '/kaggle/input/CORD-19-research-challenge/'
metadata_path = f'{root_path}/metadata.csv'
meta_df = pd.read_csv(metadata_path, low_memory=False, dtype={
    'pubmed_id': str,
    'Microsoft Academic Paper ID': str,
    'doi': str
})
print(meta_df.head())

print(meta_df.info())

# Fetch All of JSON File Path
all_json = glob.glob(f'{root_path}/**/*.json', recursive=True)
print(len(all_json))


class FileReader:
    def __init__(self, file_path):
        with open(file_path) as file:
            content = json.load(file)
            self.paper_id = content['paper_id']
            self.abstract = []
            self.body_text = []
            # Abstract
            for entry in content['abstract']:
                self.abstract.append(entry['text'])
            # Body text
            for entry in content['body_text']:
                self.body_text.append(entry['text'])
            self.abstract = '\n'.join(self.abstract)
            self.body_text = '\n'.join(self.body_text)

    def __repr__(self):
        return f'{self.paper_id}: {self.abstract[:200]}... {self.body_text[:200]}...'


first_row = FileReader(all_json[0])
print(first_row)

# Todo-------> Select just the first 200 ones.
all_json = all_json[:QUANT_PAPERS]


# Helper function adds break after every words when character length reach to certain amount.
# This is for the interactive plot so that hover tool fits the screen.


def get_breaks(content, length):
    data = ""
    words = content.split(' ')
    total_chars = 0

    # add break every length characters
    for i in range(len(words)):
        total_chars += len(words[i])
        if total_chars > length:
            data = data + "<br>" + words[i]
            total_chars = 0
        else:
            data = data + " " + words[i]
    return data


# Load the Data into DataFrame¶
dict_ = {'paper_id': [], 'doi': [], 'abstract': [], 'body_text': [], 'authors': [], 'title': [], 'journal': [],
         'abstract_summary': []}
for idx, entry in enumerate(all_json):
    if idx % (len(all_json) // 10) == 0:
        print(f'Processing index: {idx} of {len(all_json)}')

    try:
        content = FileReader(entry)
    except Exception as e:
        continue  # invalid paper format, skip

    # get metadata information
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    # no metadata, skip this paper
    if len(meta_data) == 0:
        continue

    dict_['abstract'].append(content.abstract)
    dict_['paper_id'].append(content.paper_id)
    dict_['body_text'].append(content.body_text)

    # also create a column for the summary of abstract to be used in a plot
    if len(content.abstract) == 0:
        # no abstract provided
        dict_['abstract_summary'].append("Not provided.")
    elif len(content.abstract.split(' ')) > 100:
        # abstract provided is too long for plot, take first 100 words append with ...
        info = content.abstract.split(' ')[:100]
        summary = get_breaks(' '.join(info), 40)
        dict_['abstract_summary'].append(summary + "...")
    else:
        # abstract is short enough
        summary = get_breaks(content.abstract, 40)
        dict_['abstract_summary'].append(summary)

    # get metadata information
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]

    try:
        # if more than one author
        authors = meta_data['authors'].values[0].split(';')
        if len(authors) > 2:
            # if more than 2 authors, take them all with html tag breaks in between
            dict_['authors'].append(get_breaks('. '.join(authors), 40))
        else:
            # authors will fit in plot
            dict_['authors'].append(". ".join(authors))
    except Exception as e:
        # if only one author - or Null valie
        dict_['authors'].append(meta_data['authors'].values[0])

    # add the title information, add breaks when needed
    try:
        title = get_breaks(meta_data['title'].values[0], 40)
        dict_['title'].append(title)
    # if title was not provided
    except Exception as e:
        dict_['title'].append(meta_data['title'].values[0])

    # add the journal information
    dict_['journal'].append(meta_data['journal'].values[0])

    # add doi
    dict_['doi'].append(meta_data['doi'].values[0])

df_covid = pd.DataFrame(dict_, columns=['paper_id', 'doi', 'abstract', 'body_text', 'authors', 'title', 'journal',
                                        'abstract_summary'])
print(df_covid.head())

# Adding word count columns for both abstract and body_text can be useful parameters later:
df_covid['abstract_word_count'] = df_covid['abstract'].apply(lambda x: len(x.strip().split()))  # word count in abstract
df_covid['body_word_count'] = df_covid['body_text'].apply(lambda x: len(x.strip().split()))  # word count in body
df_covid['body_unique_words'] = df_covid['body_text'].apply(
    lambda x: len(set(str(x).split())))  # number of unique words in body
print(df_covid.head())

print(df_covid.info())

print(df_covid['abstract'].describe(include='all'))

# Handle Possible Duplicates

df_covid.drop_duplicates(['abstract', 'body_text'], inplace=True)
df_covid['abstract'].describe(include='all')

df_covid['body_text'].describe(include='all')

# Take a Look at the Data
print(df_covid.head())

print(df_covid.describe())

# Data Pre-processing

# ------>In Kaggle we will limit the dataframe to 10,000 instances. Change it later

# TODO: This number should be less or equal than quant of papers
df = df_covid.sample(RANGE_INDEX, random_state=42)
del df_covid

df.dropna(inplace=True)
print(df.info())

# Handling multiple languages
from tqdm import tqdm
from langdetect import detect
from langdetect import DetectorFactory

# set seed
DetectorFactory.seed = 0

# hold label - language
languages = []

# go through each text
for ii in tqdm(range(0, len(df))):
    # split by space into list, take the first x intex, join with space
    text = df.iloc[ii]['body_text'].split(" ")

    lang = "en"
    try:
        if len(text) > 50:
            lang = detect(" ".join(text[:50]))
        elif len(text) > 0:
            lang = detect(" ".join(text[:len(text)]))
    # ught... beginning of the document was not in a good format
    except Exception as e:
        all_words = set(text)
        try:
            lang = detect(" ".join(all_words))
        # what!! :( let's see if we can find any text in abstract...
        except Exception as e:

            try:
                # let's try to label it through the abstract then
                lang = detect(df.iloc[ii]['abstract_summary'])
            except Exception as e:
                lang = "unknown"
                pass

    # get the language
    print(languages.append(lang))

from pprint import pprint

languages_dict = {}
for lang in set(languages):
    languages_dict[lang] = languages.count(lang)

print("Total: {}\n".format(len(languages)))
pprint(languages_dict)

# Lets take a look at the language distribution in the dataset
df['language'] = languages
plt.bar(range(len(languages_dict)), list(languages_dict.values()), align='center')
plt.xticks(range(len(languages_dict)), list(languages_dict.keys()))
plt.title("Distribution of Languages in Dataset")
plt.show()

df = df[df['language'] == 'en']
print(df.info())

# Download the spacy bio parser

# from IPython.utils import io
# with io.capture_output() as captured:
#    !pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.2.4/en_core_sci_lg-0.2.4.tar.gz

# NLP
import spacy  # sudo pip3 install spacy

from spacy.lang.en.stop_words import STOP_WORDS
import en_core_sci_lg  # model downloaded in previous step
# pip install scispacy
# pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.2.4/en_core_sci_lg-0.2.4.tar.gz

import string

punctuations = string.punctuation
stopwords = list(STOP_WORDS)
print(stopwords[:10])

# Now the above stopwords are used in everyday english text. Research papers will often frequently use words
# that don't actually contribute to the meaning and are not considered everyday stopwords.
custom_stop_words = [
    'doi', 'preprint', 'copyright', 'peer', 'reviewed', 'org', 'https', 'et', 'al', 'author', 'figure',
    'rights', 'reserved', 'permission', 'used', 'using', 'biorxiv', 'medrxiv', 'license', 'fig', 'fig.',
    'al.', 'Elsevier', 'PMC', 'CZI', 'www'
]

for w in custom_stop_words:
    if w not in stopwords:
        stopwords.append(w)

# Next lets create a function that will process the text data for us.
# Parser
parser = en_core_sci_lg.load(disable=["tagger", "ner"])
parser.max_length = 7000000


def spacy_tokenizer(sentence):
    mytokens = parser(sentence)
    mytokens = [word.lemma_.lower().strip() if word.lemma_ != "-PRON-" else word.lower_ for word in mytokens]
    mytokens = [word for word in mytokens if word not in stopwords and word not in punctuations]
    mytokens = " ".join([i for i in mytokens])
    return mytokens


# Applying the text-processing function on the body_text.
print(tqdm.pandas())
df["processed_text"] = df["body_text"].progress_apply(spacy_tokenizer)

# Let's take a look at word count in the papers
import seaborn as sns

sns.distplot(df['body_word_count'])
print(df['body_word_count'].describe())

sns.distplot(df['body_unique_words'])
print(df['body_unique_words'].describe())

# Vectorization

from sklearn.feature_extraction.text import TfidfVectorizer


def vectorize(text, maxx_features):
    vectorizer = TfidfVectorizer(max_features=maxx_features)
    X = vectorizer.fit_transform(text)
    return X


text = df['processed_text'].values
X = vectorize(text, 2 ** 12)
print(X.shape)

# PCA & Clustering

from sklearn.decomposition import PCA

pca = PCA(n_components=0.95, random_state=42)
X_reduced = pca.fit_transform(X.toarray())
print(X_reduced.shape)

from sklearn.cluster import KMeans

# How many clusters?

from sklearn import metrics
from scipy.spatial.distance import cdist

# run kmeans with many different k
distortions = []
K = range(2, 50)
for k in K:
    k_means = KMeans(n_clusters=k, random_state=42).fit(X_reduced)
    k_means.fit(X_reduced)
    distortions.append(sum(np.min(cdist(X_reduced, k_means.cluster_centers_, 'euclidean'), axis=1)) / X.shape[0])
    # print('Found distortion for {} clusters'.format(k))

X_line = [K[0], K[-1]]
Y_line = [distortions[0], distortions[-1]]

# Plot the elbow
plt.plot(K, distortions, 'b-')
plt.plot(X_line, Y_line, 'r')
plt.xlabel('k')
plt.ylabel('Distortion')
plt.title('The Elbow Method showing the optimal k')
plt.show()

# Run k-means
k = 20
kmeans = KMeans(n_clusters=k, random_state=42)
y_pred = kmeans.fit_predict(X_reduced)
df['y'] = y_pred

from sklearn.manifold import TSNE

tsne = TSNE(verbose=1, perplexity=100, random_state=42)
X_embedded = tsne.fit_transform(X.toarray())
############
# This looks pretty bland. There are some clusters we can immediately detect, but the many instances closer to the
# center are harder to separate. t-SNE did a good job at reducing the dimensionality, but now we need some labels.
# Let's use the clusters found by k-means as labels. This will help visually separate different concentrations of topics
############

from matplotlib import pyplot as plt
import seaborn as sns

# sns settings
sns.set(rc={'figure.figsize': (15, 15)})

# colors
palette = sns.color_palette("bright", 1)

# plot
sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1], palette=palette)
plt.title('t-SNE with no Labels')
plt.savefig("t-sne_covid19.png")
plt.show()

from matplotlib import pyplot as plt
import seaborn as sns

# sns settings
sns.set(rc={'figure.figsize': (15, 15)})

# colors
palette = sns.hls_palette(20, l=.4, s=.9)

# plot
sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1], hue=y_pred, legend='full', palette=palette)
plt.title('t-SNE with Kmeans Labels')
plt.savefig("improved_cluster_tsne.png")
plt.show()

2021fec0f416a724bfd0ee6dbca108f9cfd6d186
<mask token>


class FileReader:

    def __init__(self, file_path):
        with open(file_path) as file:
            content = json.load(file)
            self.paper_id = content['paper_id']
            self.abstract = []
            self.body_text = []
            for entry in content['abstract']:
                self.abstract.append(entry['text'])
            for entry in content['body_text']:
                self.body_text.append(entry['text'])
            self.abstract = '\n'.join(self.abstract)
            self.body_text = '\n'.join(self.body_text)

    def __repr__(self):
        return (
            f'{self.paper_id}: {self.abstract[:200]}... {self.body_text[:200]}...'
            )


<mask token>


def spacy_tokenizer(sentence):
    mytokens = parser(sentence)
    mytokens = [(word.lemma_.lower().strip() if word.lemma_ != '-PRON-' else
        word.lower_) for word in mytokens]
    mytokens = [word for word in mytokens if word not in stopwords and word
         not in punctuations]
    mytokens = ' '.join([i for i in mytokens])
    return mytokens


<mask token>


def vectorize(text, maxx_features):
    vectorizer = TfidfVectorizer(max_features=maxx_features)
    X = vectorizer.fit_transform(text)
    return X


<mask token>

<mask token>
plt.style.use('ggplot')
<mask token>
print(meta_df.head())
print(meta_df.info())
<mask token>
print(len(all_json))


class FileReader:

    def __init__(self, file_path):
        with open(file_path) as file:
            content = json.load(file)
            self.paper_id = content['paper_id']
            self.abstract = []
            self.body_text = []
            for entry in content['abstract']:
                self.abstract.append(entry['text'])
            for entry in content['body_text']:
                self.body_text.append(entry['text'])
            self.abstract = '\n'.join(self.abstract)
            self.body_text = '\n'.join(self.body_text)

    def __repr__(self):
        return (
            f'{self.paper_id}: {self.abstract[:200]}... {self.body_text[:200]}...'
            )


<mask token>
print(first_row)
<mask token>


def get_breaks(content, length):
    data = ''
    words = content.split(' ')
    total_chars = 0
    for i in range(len(words)):
        total_chars += len(words[i])
        if total_chars > length:
            data = data + '<br>' + words[i]
            total_chars = 0
        else:
            data = data + ' ' + words[i]
    return data


<mask token>
for idx, entry in enumerate(all_json):
    if idx % (len(all_json) // 10) == 0:
        print(f'Processing index: {idx} of {len(all_json)}')
    try:
        content = FileReader(entry)
    except Exception as e:
        continue
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    if len(meta_data) == 0:
        continue
    dict_['abstract'].append(content.abstract)
    dict_['paper_id'].append(content.paper_id)
    dict_['body_text'].append(content.body_text)
    if len(content.abstract) == 0:
        dict_['abstract_summary'].append('Not provided.')
    elif len(content.abstract.split(' ')) > 100:
        info = content.abstract.split(' ')[:100]
        summary = get_breaks(' '.join(info), 40)
        dict_['abstract_summary'].append(summary + '...')
    else:
        summary = get_breaks(content.abstract, 40)
        dict_['abstract_summary'].append(summary)
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    try:
        authors = meta_data['authors'].values[0].split(';')
        if len(authors) > 2:
            dict_['authors'].append(get_breaks('. '.join(authors), 40))
        else:
            dict_['authors'].append('. '.join(authors))
    except Exception as e:
        dict_['authors'].append(meta_data['authors'].values[0])
    try:
        title = get_breaks(meta_data['title'].values[0], 40)
        dict_['title'].append(title)
    except Exception as e:
        dict_['title'].append(meta_data['title'].values[0])
    dict_['journal'].append(meta_data['journal'].values[0])
    dict_['doi'].append(meta_data['doi'].values[0])
<mask token>
print(df_covid.head())
<mask token>
print(df_covid.head())
print(df_covid.info())
print(df_covid['abstract'].describe(include='all'))
df_covid.drop_duplicates(['abstract', 'body_text'], inplace=True)
df_covid['abstract'].describe(include='all')
df_covid['body_text'].describe(include='all')
print(df_covid.head())
print(df_covid.describe())
<mask token>
del df_covid
df.dropna(inplace=True)
print(df.info())
<mask token>
for ii in tqdm(range(0, len(df))):
    text = df.iloc[ii]['body_text'].split(' ')
    lang = 'en'
    try:
        if len(text) > 50:
            lang = detect(' '.join(text[:50]))
        elif len(text) > 0:
            lang = detect(' '.join(text[:len(text)]))
    except Exception as e:
        all_words = set(text)
        try:
            lang = detect(' '.join(all_words))
        except Exception as e:
            try:
                lang = detect(df.iloc[ii]['abstract_summary'])
            except Exception as e:
                lang = 'unknown'
                pass
    print(languages.append(lang))
<mask token>
for lang in set(languages):
    languages_dict[lang] = languages.count(lang)
print('Total: {}\n'.format(len(languages)))
pprint(languages_dict)
<mask token>
plt.bar(range(len(languages_dict)), list(languages_dict.values()), align=
    'center')
plt.xticks(range(len(languages_dict)), list(languages_dict.keys()))
plt.title('Distribution of Languages in Dataset')
plt.show()
<mask token>
print(df.info())
<mask token>
print(stopwords[:10])
<mask token>
for w in custom_stop_words:
    if w not in stopwords:
        stopwords.append(w)
<mask token>


def spacy_tokenizer(sentence):
    mytokens = parser(sentence)
    mytokens = [(word.lemma_.lower().strip() if word.lemma_ != '-PRON-' else
        word.lower_) for word in mytokens]
    mytokens = [word for word in mytokens if word not in stopwords and word
         not in punctuations]
    mytokens = ' '.join([i for i in mytokens])
    return mytokens


print(tqdm.pandas())
<mask token>
sns.distplot(df['body_word_count'])
print(df['body_word_count'].describe())
sns.distplot(df['body_unique_words'])
print(df['body_unique_words'].describe())
<mask token>


def vectorize(text, maxx_features):
    vectorizer = TfidfVectorizer(max_features=maxx_features)
    X = vectorizer.fit_transform(text)
    return X


<mask token>
print(X.shape)
<mask token>
print(X_reduced.shape)
<mask token>
for k in K:
    k_means = KMeans(n_clusters=k, random_state=42).fit(X_reduced)
    k_means.fit(X_reduced)
    distortions.append(sum(np.min(cdist(X_reduced, k_means.cluster_centers_,
        'euclidean'), axis=1)) / X.shape[0])
<mask token>
plt.plot(K, distortions, 'b-')
plt.plot(X_line, Y_line, 'r')
plt.xlabel('k')
plt.ylabel('Distortion')
plt.title('The Elbow Method showing the optimal k')
plt.show()
<mask token>
sns.set(rc={'figure.figsize': (15, 15)})
<mask token>
sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1], palette=palette)
plt.title('t-SNE with no Labels')
plt.savefig('t-sne_covid19.png')
plt.show()
<mask token>
sns.set(rc={'figure.figsize': (15, 15)})
<mask token>
sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1], hue=y_pred, legend=
    'full', palette=palette)
plt.title('t-SNE with Kmeans Labels')
plt.savefig('improved_cluster_tsne.png')
plt.show()

ROOT_PATH = 'C:\\Users\\amondejar\\Desktop\\SharedVM\\dataset'
QUANT_PAPERS = 50000
RANGE_INDEX = 44557
<mask token>
plt.style.use('ggplot')
root_path = ROOT_PATH
metadata_path = f'{root_path}/metadata.csv'
meta_df = pd.read_csv(metadata_path, low_memory=False, dtype={'pubmed_id':
    str, 'Microsoft Academic Paper ID': str, 'doi': str})
print(meta_df.head())
print(meta_df.info())
all_json = glob.glob(f'{root_path}/**/*.json', recursive=True)
print(len(all_json))


class FileReader:

    def __init__(self, file_path):
        with open(file_path) as file:
            content = json.load(file)
            self.paper_id = content['paper_id']
            self.abstract = []
            self.body_text = []
            for entry in content['abstract']:
                self.abstract.append(entry['text'])
            for entry in content['body_text']:
                self.body_text.append(entry['text'])
            self.abstract = '\n'.join(self.abstract)
            self.body_text = '\n'.join(self.body_text)

    def __repr__(self):
        return (
            f'{self.paper_id}: {self.abstract[:200]}... {self.body_text[:200]}...'
            )


first_row = FileReader(all_json[0])
print(first_row)
all_json = all_json[:QUANT_PAPERS]


def get_breaks(content, length):
    data = ''
    words = content.split(' ')
    total_chars = 0
    for i in range(len(words)):
        total_chars += len(words[i])
        if total_chars > length:
            data = data + '<br>' + words[i]
            total_chars = 0
        else:
            data = data + ' ' + words[i]
    return data


dict_ = {'paper_id': [], 'doi': [], 'abstract': [], 'body_text': [],
    'authors': [], 'title': [], 'journal': [], 'abstract_summary': []}
for idx, entry in enumerate(all_json):
    if idx % (len(all_json) // 10) == 0:
        print(f'Processing index: {idx} of {len(all_json)}')
    try:
        content = FileReader(entry)
    except Exception as e:
        continue
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    if len(meta_data) == 0:
        continue
    dict_['abstract'].append(content.abstract)
    dict_['paper_id'].append(content.paper_id)
    dict_['body_text'].append(content.body_text)
    if len(content.abstract) == 0:
        dict_['abstract_summary'].append('Not provided.')
    elif len(content.abstract.split(' ')) > 100:
        info = content.abstract.split(' ')[:100]
        summary = get_breaks(' '.join(info), 40)
        dict_['abstract_summary'].append(summary + '...')
    else:
        summary = get_breaks(content.abstract, 40)
        dict_['abstract_summary'].append(summary)
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    try:
        authors = meta_data['authors'].values[0].split(';')
        if len(authors) > 2:
            dict_['authors'].append(get_breaks('. '.join(authors), 40))
        else:
            dict_['authors'].append('. '.join(authors))
    except Exception as e:
        dict_['authors'].append(meta_data['authors'].values[0])
    try:
        title = get_breaks(meta_data['title'].values[0], 40)
        dict_['title'].append(title)
    except Exception as e:
        dict_['title'].append(meta_data['title'].values[0])
    dict_['journal'].append(meta_data['journal'].values[0])
    dict_['doi'].append(meta_data['doi'].values[0])
df_covid = pd.DataFrame(dict_, columns=['paper_id', 'doi', 'abstract',
    'body_text', 'authors', 'title', 'journal', 'abstract_summary'])
print(df_covid.head())
df_covid['abstract_word_count'] = df_covid['abstract'].apply(lambda x: len(
    x.strip().split()))
df_covid['body_word_count'] = df_covid['body_text'].apply(lambda x: len(x.
    strip().split()))
df_covid['body_unique_words'] = df_covid['body_text'].apply(lambda x: len(
    set(str(x).split())))
print(df_covid.head())
print(df_covid.info())
print(df_covid['abstract'].describe(include='all'))
df_covid.drop_duplicates(['abstract', 'body_text'], inplace=True)
df_covid['abstract'].describe(include='all')
df_covid['body_text'].describe(include='all')
print(df_covid.head())
print(df_covid.describe())
df = df_covid.sample(RANGE_INDEX, random_state=42)
del df_covid
df.dropna(inplace=True)
print(df.info())
<mask token>
DetectorFactory.seed = 0
languages = []
for ii in tqdm(range(0, len(df))):
    text = df.iloc[ii]['body_text'].split(' ')
    lang = 'en'
    try:
        if len(text) > 50:
            lang = detect(' '.join(text[:50]))
        elif len(text) > 0:
            lang = detect(' '.join(text[:len(text)]))
    except Exception as e:
        all_words = set(text)
        try:
            lang = detect(' '.join(all_words))
        except Exception as e:
            try:
                lang = detect(df.iloc[ii]['abstract_summary'])
            except Exception as e:
                lang = 'unknown'
                pass
    print(languages.append(lang))
<mask token>
languages_dict = {}
for lang in set(languages):
    languages_dict[lang] = languages.count(lang)
print('Total: {}\n'.format(len(languages)))
pprint(languages_dict)
df['language'] = languages
plt.bar(range(len(languages_dict)), list(languages_dict.values()), align=
    'center')
plt.xticks(range(len(languages_dict)), list(languages_dict.keys()))
plt.title('Distribution of Languages in Dataset')
plt.show()
df = df[df['language'] == 'en']
print(df.info())
<mask token>
punctuations = string.punctuation
stopwords = list(STOP_WORDS)
print(stopwords[:10])
custom_stop_words = ['doi', 'preprint', 'copyright', 'peer', 'reviewed',
    'org', 'https', 'et', 'al', 'author', 'figure', 'rights', 'reserved',
    'permission', 'used', 'using', 'biorxiv', 'medrxiv', 'license', 'fig',
    'fig.', 'al.', 'Elsevier', 'PMC', 'CZI', 'www']
for w in custom_stop_words:
    if w not in stopwords:
        stopwords.append(w)
parser = en_core_sci_lg.load(disable=['tagger', 'ner'])
parser.max_length = 7000000


def spacy_tokenizer(sentence):
    mytokens = parser(sentence)
    mytokens = [(word.lemma_.lower().strip() if word.lemma_ != '-PRON-' else
        word.lower_) for word in mytokens]
    mytokens = [word for word in mytokens if word not in stopwords and word
         not in punctuations]
    mytokens = ' '.join([i for i in mytokens])
    return mytokens


print(tqdm.pandas())
df['processed_text'] = df['body_text'].progress_apply(spacy_tokenizer)
<mask token>
sns.distplot(df['body_word_count'])
print(df['body_word_count'].describe())
sns.distplot(df['body_unique_words'])
print(df['body_unique_words'].describe())
<mask token>


def vectorize(text, maxx_features):
    vectorizer = TfidfVectorizer(max_features=maxx_features)
    X = vectorizer.fit_transform(text)
    return X


text = df['processed_text'].values
X = vectorize(text, 2 ** 12)
print(X.shape)
<mask token>
pca = PCA(n_components=0.95, random_state=42)
X_reduced = pca.fit_transform(X.toarray())
print(X_reduced.shape)
<mask token>
distortions = []
K = range(2, 50)
for k in K:
    k_means = KMeans(n_clusters=k, random_state=42).fit(X_reduced)
    k_means.fit(X_reduced)
    distortions.append(sum(np.min(cdist(X_reduced, k_means.cluster_centers_,
        'euclidean'), axis=1)) / X.shape[0])
X_line = [K[0], K[-1]]
Y_line = [distortions[0], distortions[-1]]
plt.plot(K, distortions, 'b-')
plt.plot(X_line, Y_line, 'r')
plt.xlabel('k')
plt.ylabel('Distortion')
plt.title('The Elbow Method showing the optimal k')
plt.show()
k = 20
kmeans = KMeans(n_clusters=k, random_state=42)
y_pred = kmeans.fit_predict(X_reduced)
df['y'] = y_pred
<mask token>
tsne = TSNE(verbose=1, perplexity=100, random_state=42)
X_embedded = tsne.fit_transform(X.toarray())
<mask token>
sns.set(rc={'figure.figsize': (15, 15)})
palette = sns.color_palette('bright', 1)
sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1], palette=palette)
plt.title('t-SNE with no Labels')
plt.savefig('t-sne_covid19.png')
plt.show()
<mask token>
sns.set(rc={'figure.figsize': (15, 15)})
palette = sns.hls_palette(20, l=0.4, s=0.9)
sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1], hue=y_pred, legend=
    'full', palette=palette)
plt.title('t-SNE with Kmeans Labels')
plt.savefig('improved_cluster_tsne.png')
plt.show()

ROOT_PATH = 'C:\\Users\\amondejar\\Desktop\\SharedVM\\dataset'
QUANT_PAPERS = 50000
RANGE_INDEX = 44557
import numpy as np
import pandas as pd
import glob
import json
import matplotlib.pyplot as plt
plt.style.use('ggplot')
root_path = ROOT_PATH
metadata_path = f'{root_path}/metadata.csv'
meta_df = pd.read_csv(metadata_path, low_memory=False, dtype={'pubmed_id':
    str, 'Microsoft Academic Paper ID': str, 'doi': str})
print(meta_df.head())
print(meta_df.info())
all_json = glob.glob(f'{root_path}/**/*.json', recursive=True)
print(len(all_json))


class FileReader:

    def __init__(self, file_path):
        with open(file_path) as file:
            content = json.load(file)
            self.paper_id = content['paper_id']
            self.abstract = []
            self.body_text = []
            for entry in content['abstract']:
                self.abstract.append(entry['text'])
            for entry in content['body_text']:
                self.body_text.append(entry['text'])
            self.abstract = '\n'.join(self.abstract)
            self.body_text = '\n'.join(self.body_text)

    def __repr__(self):
        return (
            f'{self.paper_id}: {self.abstract[:200]}... {self.body_text[:200]}...'
            )


first_row = FileReader(all_json[0])
print(first_row)
all_json = all_json[:QUANT_PAPERS]


def get_breaks(content, length):
    data = ''
    words = content.split(' ')
    total_chars = 0
    for i in range(len(words)):
        total_chars += len(words[i])
        if total_chars > length:
            data = data + '<br>' + words[i]
            total_chars = 0
        else:
            data = data + ' ' + words[i]
    return data


dict_ = {'paper_id': [], 'doi': [], 'abstract': [], 'body_text': [],
    'authors': [], 'title': [], 'journal': [], 'abstract_summary': []}
for idx, entry in enumerate(all_json):
    if idx % (len(all_json) // 10) == 0:
        print(f'Processing index: {idx} of {len(all_json)}')
    try:
        content = FileReader(entry)
    except Exception as e:
        continue
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    if len(meta_data) == 0:
        continue
    dict_['abstract'].append(content.abstract)
    dict_['paper_id'].append(content.paper_id)
    dict_['body_text'].append(content.body_text)
    if len(content.abstract) == 0:
        dict_['abstract_summary'].append('Not provided.')
    elif len(content.abstract.split(' ')) > 100:
        info = content.abstract.split(' ')[:100]
        summary = get_breaks(' '.join(info), 40)
        dict_['abstract_summary'].append(summary + '...')
    else:
        summary = get_breaks(content.abstract, 40)
        dict_['abstract_summary'].append(summary)
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    try:
        authors = meta_data['authors'].values[0].split(';')
        if len(authors) > 2:
            dict_['authors'].append(get_breaks('. '.join(authors), 40))
        else:
            dict_['authors'].append('. '.join(authors))
    except Exception as e:
        dict_['authors'].append(meta_data['authors'].values[0])
    try:
        title = get_breaks(meta_data['title'].values[0], 40)
        dict_['title'].append(title)
    except Exception as e:
        dict_['title'].append(meta_data['title'].values[0])
    dict_['journal'].append(meta_data['journal'].values[0])
    dict_['doi'].append(meta_data['doi'].values[0])
df_covid = pd.DataFrame(dict_, columns=['paper_id', 'doi', 'abstract',
    'body_text', 'authors', 'title', 'journal', 'abstract_summary'])
print(df_covid.head())
df_covid['abstract_word_count'] = df_covid['abstract'].apply(lambda x: len(
    x.strip().split()))
df_covid['body_word_count'] = df_covid['body_text'].apply(lambda x: len(x.
    strip().split()))
df_covid['body_unique_words'] = df_covid['body_text'].apply(lambda x: len(
    set(str(x).split())))
print(df_covid.head())
print(df_covid.info())
print(df_covid['abstract'].describe(include='all'))
df_covid.drop_duplicates(['abstract', 'body_text'], inplace=True)
df_covid['abstract'].describe(include='all')
df_covid['body_text'].describe(include='all')
print(df_covid.head())
print(df_covid.describe())
df = df_covid.sample(RANGE_INDEX, random_state=42)
del df_covid
df.dropna(inplace=True)
print(df.info())
from tqdm import tqdm
from langdetect import detect
from langdetect import DetectorFactory
DetectorFactory.seed = 0
languages = []
for ii in tqdm(range(0, len(df))):
    text = df.iloc[ii]['body_text'].split(' ')
    lang = 'en'
    try:
        if len(text) > 50:
            lang = detect(' '.join(text[:50]))
        elif len(text) > 0:
            lang = detect(' '.join(text[:len(text)]))
    except Exception as e:
        all_words = set(text)
        try:
            lang = detect(' '.join(all_words))
        except Exception as e:
            try:
                lang = detect(df.iloc[ii]['abstract_summary'])
            except Exception as e:
                lang = 'unknown'
                pass
    print(languages.append(lang))
from pprint import pprint
languages_dict = {}
for lang in set(languages):
    languages_dict[lang] = languages.count(lang)
print('Total: {}\n'.format(len(languages)))
pprint(languages_dict)
df['language'] = languages
plt.bar(range(len(languages_dict)), list(languages_dict.values()), align=
    'center')
plt.xticks(range(len(languages_dict)), list(languages_dict.keys()))
plt.title('Distribution of Languages in Dataset')
plt.show()
df = df[df['language'] == 'en']
print(df.info())
import spacy
from spacy.lang.en.stop_words import STOP_WORDS
import en_core_sci_lg
import string
punctuations = string.punctuation
stopwords = list(STOP_WORDS)
print(stopwords[:10])
custom_stop_words = ['doi', 'preprint', 'copyright', 'peer', 'reviewed',
    'org', 'https', 'et', 'al', 'author', 'figure', 'rights', 'reserved',
    'permission', 'used', 'using', 'biorxiv', 'medrxiv', 'license', 'fig',
    'fig.', 'al.', 'Elsevier', 'PMC', 'CZI', 'www']
for w in custom_stop_words:
    if w not in stopwords:
        stopwords.append(w)
parser = en_core_sci_lg.load(disable=['tagger', 'ner'])
parser.max_length = 7000000


def spacy_tokenizer(sentence):
    mytokens = parser(sentence)
    mytokens = [(word.lemma_.lower().strip() if word.lemma_ != '-PRON-' else
        word.lower_) for word in mytokens]
    mytokens = [word for word in mytokens if word not in stopwords and word
         not in punctuations]
    mytokens = ' '.join([i for i in mytokens])
    return mytokens


print(tqdm.pandas())
df['processed_text'] = df['body_text'].progress_apply(spacy_tokenizer)
import seaborn as sns
sns.distplot(df['body_word_count'])
print(df['body_word_count'].describe())
sns.distplot(df['body_unique_words'])
print(df['body_unique_words'].describe())
from sklearn.feature_extraction.text import TfidfVectorizer


def vectorize(text, maxx_features):
    vectorizer = TfidfVectorizer(max_features=maxx_features)
    X = vectorizer.fit_transform(text)
    return X


text = df['processed_text'].values
X = vectorize(text, 2 ** 12)
print(X.shape)
from sklearn.decomposition import PCA
pca = PCA(n_components=0.95, random_state=42)
X_reduced = pca.fit_transform(X.toarray())
print(X_reduced.shape)
from sklearn.cluster import KMeans
from sklearn import metrics
from scipy.spatial.distance import cdist
distortions = []
K = range(2, 50)
for k in K:
    k_means = KMeans(n_clusters=k, random_state=42).fit(X_reduced)
    k_means.fit(X_reduced)
    distortions.append(sum(np.min(cdist(X_reduced, k_means.cluster_centers_,
        'euclidean'), axis=1)) / X.shape[0])
X_line = [K[0], K[-1]]
Y_line = [distortions[0], distortions[-1]]
plt.plot(K, distortions, 'b-')
plt.plot(X_line, Y_line, 'r')
plt.xlabel('k')
plt.ylabel('Distortion')
plt.title('The Elbow Method showing the optimal k')
plt.show()
k = 20
kmeans = KMeans(n_clusters=k, random_state=42)
y_pred = kmeans.fit_predict(X_reduced)
df['y'] = y_pred
from sklearn.manifold import TSNE
tsne = TSNE(verbose=1, perplexity=100, random_state=42)
X_embedded = tsne.fit_transform(X.toarray())
from matplotlib import pyplot as plt
import seaborn as sns
sns.set(rc={'figure.figsize': (15, 15)})
palette = sns.color_palette('bright', 1)
sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1], palette=palette)
plt.title('t-SNE with no Labels')
plt.savefig('t-sne_covid19.png')
plt.show()
from matplotlib import pyplot as plt
import seaborn as sns
sns.set(rc={'figure.figsize': (15, 15)})
palette = sns.hls_palette(20, l=0.4, s=0.9)
sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1], hue=y_pred, legend=
    'full', palette=palette)
plt.title('t-SNE with Kmeans Labels')
plt.savefig('improved_cluster_tsne.png')
plt.show()

# Main source: https://www.kaggle.com/maksimeren/covid-19-literature-clustering

ROOT_PATH = 'C:\\Users\\amondejar\\Desktop\\SharedVM\\dataset'
QUANT_PAPERS = 50000
RANGE_INDEX = 44557
# RangeIndex samples:
# For 10000 papers, we have 8892 entries
# For 1000 papers, we have 898 entries
# For 500 papers, we have 454 entries
# For 200 papers, we have 180 entries
# More details? See line 188 and see what we have in variable 'count'

######################################################################

import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)
import glob
import json
import matplotlib.pyplot as plt

plt.style.use('ggplot')

# Let's load the metadata of the dateset. 'title' and 'journal' attributes may be useful later when we cluster the
# articles to see what kinds of articles cluster together.


root_path = ROOT_PATH
# root_path = '/kaggle/input/CORD-19-research-challenge/'
metadata_path = f'{root_path}/metadata.csv'
meta_df = pd.read_csv(metadata_path, low_memory=False, dtype={
    'pubmed_id': str,
    'Microsoft Academic Paper ID': str,
    'doi': str
})
print(meta_df.head())

print(meta_df.info())

# Fetch All of JSON File Path
all_json = glob.glob(f'{root_path}/**/*.json', recursive=True)
print(len(all_json))


class FileReader:
    def __init__(self, file_path):
        with open(file_path) as file:
            content = json.load(file)
            self.paper_id = content['paper_id']
            self.abstract = []
            self.body_text = []
            # Abstract
            for entry in content['abstract']:
                self.abstract.append(entry['text'])
            # Body text
            for entry in content['body_text']:
                self.body_text.append(entry['text'])
            self.abstract = '\n'.join(self.abstract)
            self.body_text = '\n'.join(self.body_text)

    def __repr__(self):
        return f'{self.paper_id}: {self.abstract[:200]}... {self.body_text[:200]}...'


first_row = FileReader(all_json[0])
print(first_row)

# Todo-------> Select just the first 200 ones.
all_json = all_json[:QUANT_PAPERS]


# Helper function adds break after every words when character length reach to certain amount.
# This is for the interactive plot so that hover tool fits the screen.


def get_breaks(content, length):
    data = ""
    words = content.split(' ')
    total_chars = 0

    # add break every length characters
    for i in range(len(words)):
        total_chars += len(words[i])
        if total_chars > length:
            data = data + "<br>" + words[i]
            total_chars = 0
        else:
            data = data + " " + words[i]
    return data


# Load the Data into DataFrame¶
dict_ = {'paper_id': [], 'doi': [], 'abstract': [], 'body_text': [], 'authors': [], 'title': [], 'journal': [],
         'abstract_summary': []}
for idx, entry in enumerate(all_json):
    if idx % (len(all_json) // 10) == 0:
        print(f'Processing index: {idx} of {len(all_json)}')

    try:
        content = FileReader(entry)
    except Exception as e:
        continue  # invalid paper format, skip

    # get metadata information
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    # no metadata, skip this paper
    if len(meta_data) == 0:
        continue

    dict_['abstract'].append(content.abstract)
    dict_['paper_id'].append(content.paper_id)
    dict_['body_text'].append(content.body_text)

    # also create a column for the summary of abstract to be used in a plot
    if len(content.abstract) == 0:
        # no abstract provided
        dict_['abstract_summary'].append("Not provided.")
    elif len(content.abstract.split(' ')) > 100:
        # abstract provided is too long for plot, take first 100 words append with ...
        info = content.abstract.split(' ')[:100]
        summary = get_breaks(' '.join(info), 40)
        dict_['abstract_summary'].append(summary + "...")
    else:
        # abstract is short enough
        summary = get_breaks(content.abstract, 40)
        dict_['abstract_summary'].append(summary)

    # get metadata information
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]

    try:
        # if more than one author
        authors = meta_data['authors'].values[0].split(';')
        if len(authors) > 2:
            # if more than 2 authors, take them all with html tag breaks in between
            dict_['authors'].append(get_breaks('. '.join(authors), 40))
        else:
            # authors will fit in plot
            dict_['authors'].append(". ".join(authors))
    except Exception as e:
        # if only one author - or Null valie
        dict_['authors'].append(meta_data['authors'].values[0])

    # add the title information, add breaks when needed
    try:
        title = get_breaks(meta_data['title'].values[0], 40)
        dict_['title'].append(title)
    # if title was not provided
    except Exception as e:
        dict_['title'].append(meta_data['title'].values[0])

    # add the journal information
    dict_['journal'].append(meta_data['journal'].values[0])

    # add doi
    dict_['doi'].append(meta_data['doi'].values[0])

df_covid = pd.DataFrame(dict_, columns=['paper_id', 'doi', 'abstract', 'body_text', 'authors', 'title', 'journal',
                                        'abstract_summary'])
print(df_covid.head())

# Adding word count columns for both abstract and body_text can be useful parameters later:
df_covid['abstract_word_count'] = df_covid['abstract'].apply(lambda x: len(x.strip().split()))  # word count in abstract
df_covid['body_word_count'] = df_covid['body_text'].apply(lambda x: len(x.strip().split()))  # word count in body
df_covid['body_unique_words'] = df_covid['body_text'].apply(
    lambda x: len(set(str(x).split())))  # number of unique words in body
print(df_covid.head())

print(df_covid.info())

print(df_covid['abstract'].describe(include='all'))

# Handle Possible Duplicates

df_covid.drop_duplicates(['abstract', 'body_text'], inplace=True)
df_covid['abstract'].describe(include='all')

df_covid['body_text'].describe(include='all')

# Take a Look at the Data
print(df_covid.head())

print(df_covid.describe())

# Data Pre-processing

# ------>In Kaggle we will limit the dataframe to 10,000 instances. Change it later

# TODO: This number should be less or equal than quant of papers
df = df_covid.sample(RANGE_INDEX, random_state=42)
del df_covid

df.dropna(inplace=True)
print(df.info())

# Handling multiple languages
from tqdm import tqdm
from langdetect import detect
from langdetect import DetectorFactory

# set seed
DetectorFactory.seed = 0

# hold label - language
languages = []

# go through each text
for ii in tqdm(range(0, len(df))):
    # split by space into list, take the first x intex, join with space
    text = df.iloc[ii]['body_text'].split(" ")

    lang = "en"
    try:
        if len(text) > 50:
            lang = detect(" ".join(text[:50]))
        elif len(text) > 0:
            lang = detect(" ".join(text[:len(text)]))
    # ught... beginning of the document was not in a good format
    except Exception as e:
        all_words = set(text)
        try:
            lang = detect(" ".join(all_words))
        # what!! :( let's see if we can find any text in abstract...
        except Exception as e:

            try:
                # let's try to label it through the abstract then
                lang = detect(df.iloc[ii]['abstract_summary'])
            except Exception as e:
                lang = "unknown"
                pass

    # get the language
    print(languages.append(lang))

from pprint import pprint

languages_dict = {}
for lang in set(languages):
    languages_dict[lang] = languages.count(lang)

print("Total: {}\n".format(len(languages)))
pprint(languages_dict)

# Lets take a look at the language distribution in the dataset
df['language'] = languages
plt.bar(range(len(languages_dict)), list(languages_dict.values()), align='center')
plt.xticks(range(len(languages_dict)), list(languages_dict.keys()))
plt.title("Distribution of Languages in Dataset")
plt.show()

df = df[df['language'] == 'en']
print(df.info())

# Download the spacy bio parser

# from IPython.utils import io
# with io.capture_output() as captured:
#    !pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.2.4/en_core_sci_lg-0.2.4.tar.gz

# NLP
import spacy  # sudo pip3 install spacy

from spacy.lang.en.stop_words import STOP_WORDS
import en_core_sci_lg  # model downloaded in previous step
# pip install scispacy
# pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.2.4/en_core_sci_lg-0.2.4.tar.gz

import string

punctuations = string.punctuation
stopwords = list(STOP_WORDS)
print(stopwords[:10])

# Now the above stopwords are used in everyday english text. Research papers will often frequently use words
# that don't actually contribute to the meaning and are not considered everyday stopwords.
custom_stop_words = [
    'doi', 'preprint', 'copyright', 'peer', 'reviewed', 'org', 'https', 'et', 'al', 'author', 'figure',
    'rights', 'reserved', 'permission', 'used', 'using', 'biorxiv', 'medrxiv', 'license', 'fig', 'fig.',
    'al.', 'Elsevier', 'PMC', 'CZI', 'www'
]

for w in custom_stop_words:
    if w not in stopwords:
        stopwords.append(w)

# Next lets create a function that will process the text data for us.
# Parser
parser = en_core_sci_lg.load(disable=["tagger", "ner"])
parser.max_length = 7000000


def spacy_tokenizer(sentence):
    mytokens = parser(sentence)
    mytokens = [word.lemma_.lower().strip() if word.lemma_ != "-PRON-" else word.lower_ for word in mytokens]
    mytokens = [word for word in mytokens if word not in stopwords and word not in punctuations]
    mytokens = " ".join([i for i in mytokens])
    return mytokens


# Applying the text-processing function on the body_text.
print(tqdm.pandas())
df["processed_text"] = df["body_text"].progress_apply(spacy_tokenizer)

# Let's take a look at word count in the papers
import seaborn as sns

sns.distplot(df['body_word_count'])
print(df['body_word_count'].describe())

sns.distplot(df['body_unique_words'])
print(df['body_unique_words'].describe())

# Vectorization

from sklearn.feature_extraction.text import TfidfVectorizer


def vectorize(text, maxx_features):
    vectorizer = TfidfVectorizer(max_features=maxx_features)
    X = vectorizer.fit_transform(text)
    return X


text = df['processed_text'].values
X = vectorize(text, 2 ** 12)
print(X.shape)

# PCA & Clustering

from sklearn.decomposition import PCA

pca = PCA(n_components=0.95, random_state=42)
X_reduced = pca.fit_transform(X.toarray())
print(X_reduced.shape)

from sklearn.cluster import KMeans

# How many clusters?

from sklearn import metrics
from scipy.spatial.distance import cdist

# run kmeans with many different k
distortions = []
K = range(2, 50)
for k in K:
    k_means = KMeans(n_clusters=k, random_state=42).fit(X_reduced)
    k_means.fit(X_reduced)
    distortions.append(sum(np.min(cdist(X_reduced, k_means.cluster_centers_, 'euclidean'), axis=1)) / X.shape[0])
    # print('Found distortion for {} clusters'.format(k))

X_line = [K[0], K[-1]]
Y_line = [distortions[0], distortions[-1]]

# Plot the elbow
plt.plot(K, distortions, 'b-')
plt.plot(X_line, Y_line, 'r')
plt.xlabel('k')
plt.ylabel('Distortion')
plt.title('The Elbow Method showing the optimal k')
plt.show()

# Run k-means
k = 20
kmeans = KMeans(n_clusters=k, random_state=42)
y_pred = kmeans.fit_predict(X_reduced)
df['y'] = y_pred

from sklearn.manifold import TSNE

tsne = TSNE(verbose=1, perplexity=100, random_state=42)
X_embedded = tsne.fit_transform(X.toarray())
############
# This looks pretty bland. There are some clusters we can immediately detect, but the many instances closer to the
# center are harder to separate. t-SNE did a good job at reducing the dimensionality, but now we need some labels.
# Let's use the clusters found by k-means as labels. This will help visually separate different concentrations of topics
############

from matplotlib import pyplot as plt
import seaborn as sns

# sns settings
sns.set(rc={'figure.figsize': (15, 15)})

# colors
palette = sns.color_palette("bright", 1)

# plot
sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1], palette=palette)
plt.title('t-SNE with no Labels')
plt.savefig("t-sne_covid19.png")
plt.show()

from matplotlib import pyplot as plt
import seaborn as sns

# sns settings
sns.set(rc={'figure.figsize': (15, 15)})

# colors
palette = sns.hls_palette(20, l=.4, s=.9)

# plot
sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1], hue=y_pred, legend='full', palette=palette)
plt.title('t-SNE with Kmeans Labels')
plt.savefig("improved_cluster_tsne.png")
plt.show()
