import re
import numpy as np
import pandas as pd
from pprint import pprint

# Gensim
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess, lemmatize
from gensim.models import CoherenceModel

# spacy for lemmatization
import spacy

import pyLDAvis
import pyLDAvis.gensim
import matplotlib.pyplot as plt

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

# NTLK Stop words
from stop_words import get_stop_words
en_stop = get_stop_words('en')


def sent_to_words(sentences):
    for sentence in sentences:
        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True)) #deacc removes any punctuation


def remove_stopwords(texts):
    return [[word for word in simple_preprocess(str(doc)) if word not in en_stop] for doc in texts]

def make_bigrams(texts):
    return [bigram_mod[doc] for doc in texts]

def make_trigrams(texts):
    return [trigram_mod[bigram_mod[doc]] for doc in texts]

def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    # lemmatization groups together inflicted forms of a verb so it can be analyzed as a single item
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in texts:
        doc = nlp(" ".join(sent))
        texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])
    return texts_out


df = pd.read_json('C:\\Users\\atapia26\\Desktop\\Things for college\\Spring 2020\\Computer Science\\CS 398\\clean_textv3.json')
print(df.text.unique())
print(df.head())

# convert to list
data = df.text.values.tolist()

pprint(data[1])

# remove URLs, @user, and hashtags from the tweet text
data = [' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ", sent).split()) for sent in data]
pprint(data[1])

data_words = list(sent_to_words(data))
sample_data_words = data_words[:10000] # contains only 10000 tweets
print(data_words[:4])

bigram = gensim.models.Phrases(sample_data_words, min_count = 5, threshold = 100)
trigram = gensim.models.Phrases(bigram[sample_data_words], threshold=100)

bigram_mod = gensim.models.phrases.Phraser(bigram)
trigram_mod = gensim.models.phrases.Phraser(trigram)

print(trigram_mod[bigram_mod[sample_data_words[1]]])

data_words_nostops = remove_stopwords(sample_data_words) # remove any unnecessary stop words
data_words_bigrams = make_bigrams(data_words_nostops) # form bigrams

nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])

# lemmatized tweets
data_lemmatized = lemmatization(data_words_bigrams, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])

print(data_lemmatized[1])

# create the dictionary and corpus needed for the topic modeling
dicOfTweets = corpora.Dictionary(data_lemmatized)

tweetTexts = data_lemmatized
# creates the bag of words (term document frequency)
corpus = [dicOfTweets.doc2bow(text) for text in tweetTexts]
# this is to make the dictionary more readable
# corpus = [[(dicOfTweets[id], freq) for id, freq in cp] for cp in corpus]
print(corpus[1])
alphas = np.ones(10)/10 # have 10 topics and I likely believe all these topics are equally likely
lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dicOfTweets, num_topics=10,
                                            random_state=100, update_every=0, chunksize=500,
                                            passes=5, alpha=alphas, per_word_topics=True, distributed=False)
pprint(lda_model.print_topics())
#pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, dicOfTweets)

8d610cb7c6c3524bb63f53de0dd0c4a50d3170c2
<mask token>


def sent_to_words(sentences):
    for sentence in sentences:
        yield gensim.utils.simple_preprocess(str(sentence), deacc=True)


def remove_stopwords(texts):
    return [[word for word in simple_preprocess(str(doc)) if word not in
        en_stop] for doc in texts]


def make_bigrams(texts):
    return [bigram_mod[doc] for doc in texts]


def make_trigrams(texts):
    return [trigram_mod[bigram_mod[doc]] for doc in texts]


def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in texts:
        doc = nlp(' '.join(sent))
        texts_out.append([token.lemma_ for token in doc if token.pos_ in
            allowed_postags])
    return texts_out


<mask token>

<mask token>
warnings.filterwarnings('ignore', category=DeprecationWarning)
<mask token>


def sent_to_words(sentences):
    for sentence in sentences:
        yield gensim.utils.simple_preprocess(str(sentence), deacc=True)


def remove_stopwords(texts):
    return [[word for word in simple_preprocess(str(doc)) if word not in
        en_stop] for doc in texts]


def make_bigrams(texts):
    return [bigram_mod[doc] for doc in texts]


def make_trigrams(texts):
    return [trigram_mod[bigram_mod[doc]] for doc in texts]


def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in texts:
        doc = nlp(' '.join(sent))
        texts_out.append([token.lemma_ for token in doc if token.pos_ in
            allowed_postags])
    return texts_out


<mask token>
print(df.text.unique())
print(df.head())
<mask token>
pprint(data[1])
<mask token>
pprint(data[1])
<mask token>
print(data_words[:4])
<mask token>
print(trigram_mod[bigram_mod[sample_data_words[1]]])
<mask token>
print(data_lemmatized[1])
<mask token>
print(corpus[1])
<mask token>
pprint(lda_model.print_topics())
<mask token>

<mask token>
warnings.filterwarnings('ignore', category=DeprecationWarning)
<mask token>
en_stop = get_stop_words('en')


def sent_to_words(sentences):
    for sentence in sentences:
        yield gensim.utils.simple_preprocess(str(sentence), deacc=True)


def remove_stopwords(texts):
    return [[word for word in simple_preprocess(str(doc)) if word not in
        en_stop] for doc in texts]


def make_bigrams(texts):
    return [bigram_mod[doc] for doc in texts]


def make_trigrams(texts):
    return [trigram_mod[bigram_mod[doc]] for doc in texts]


def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in texts:
        doc = nlp(' '.join(sent))
        texts_out.append([token.lemma_ for token in doc if token.pos_ in
            allowed_postags])
    return texts_out


df = pd.read_json(
    'C:\\Users\\atapia26\\Desktop\\Things for college\\Spring 2020\\Computer Science\\CS 398\\clean_textv3.json'
    )
print(df.text.unique())
print(df.head())
data = df.text.values.tolist()
pprint(data[1])
data = [' '.join(re.sub(
    '(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\\w+:\\/\\/\\S+)', ' ', sent).split
    ()) for sent in data]
pprint(data[1])
data_words = list(sent_to_words(data))
sample_data_words = data_words[:10000]
print(data_words[:4])
bigram = gensim.models.Phrases(sample_data_words, min_count=5, threshold=100)
trigram = gensim.models.Phrases(bigram[sample_data_words], threshold=100)
bigram_mod = gensim.models.phrases.Phraser(bigram)
trigram_mod = gensim.models.phrases.Phraser(trigram)
print(trigram_mod[bigram_mod[sample_data_words[1]]])
data_words_nostops = remove_stopwords(sample_data_words)
data_words_bigrams = make_bigrams(data_words_nostops)
nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])
data_lemmatized = lemmatization(data_words_bigrams, allowed_postags=['NOUN',
    'ADJ', 'VERB', 'ADV'])
print(data_lemmatized[1])
dicOfTweets = corpora.Dictionary(data_lemmatized)
tweetTexts = data_lemmatized
corpus = [dicOfTweets.doc2bow(text) for text in tweetTexts]
print(corpus[1])
alphas = np.ones(10) / 10
lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=
    dicOfTweets, num_topics=10, random_state=100, update_every=0, chunksize
    =500, passes=5, alpha=alphas, per_word_topics=True, distributed=False)
pprint(lda_model.print_topics())
vis = pyLDAvis.gensim.prepare(lda_model, corpus, dicOfTweets)

import re
import numpy as np
import pandas as pd
from pprint import pprint
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess, lemmatize
from gensim.models import CoherenceModel
import spacy
import pyLDAvis
import pyLDAvis.gensim
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore', category=DeprecationWarning)
from stop_words import get_stop_words
en_stop = get_stop_words('en')


def sent_to_words(sentences):
    for sentence in sentences:
        yield gensim.utils.simple_preprocess(str(sentence), deacc=True)


def remove_stopwords(texts):
    return [[word for word in simple_preprocess(str(doc)) if word not in
        en_stop] for doc in texts]


def make_bigrams(texts):
    return [bigram_mod[doc] for doc in texts]


def make_trigrams(texts):
    return [trigram_mod[bigram_mod[doc]] for doc in texts]


def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in texts:
        doc = nlp(' '.join(sent))
        texts_out.append([token.lemma_ for token in doc if token.pos_ in
            allowed_postags])
    return texts_out


df = pd.read_json(
    'C:\\Users\\atapia26\\Desktop\\Things for college\\Spring 2020\\Computer Science\\CS 398\\clean_textv3.json'
    )
print(df.text.unique())
print(df.head())
data = df.text.values.tolist()
pprint(data[1])
data = [' '.join(re.sub(
    '(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\\w+:\\/\\/\\S+)', ' ', sent).split
    ()) for sent in data]
pprint(data[1])
data_words = list(sent_to_words(data))
sample_data_words = data_words[:10000]
print(data_words[:4])
bigram = gensim.models.Phrases(sample_data_words, min_count=5, threshold=100)
trigram = gensim.models.Phrases(bigram[sample_data_words], threshold=100)
bigram_mod = gensim.models.phrases.Phraser(bigram)
trigram_mod = gensim.models.phrases.Phraser(trigram)
print(trigram_mod[bigram_mod[sample_data_words[1]]])
data_words_nostops = remove_stopwords(sample_data_words)
data_words_bigrams = make_bigrams(data_words_nostops)
nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])
data_lemmatized = lemmatization(data_words_bigrams, allowed_postags=['NOUN',
    'ADJ', 'VERB', 'ADV'])
print(data_lemmatized[1])
dicOfTweets = corpora.Dictionary(data_lemmatized)
tweetTexts = data_lemmatized
corpus = [dicOfTweets.doc2bow(text) for text in tweetTexts]
print(corpus[1])
alphas = np.ones(10) / 10
lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=
    dicOfTweets, num_topics=10, random_state=100, update_every=0, chunksize
    =500, passes=5, alpha=alphas, per_word_topics=True, distributed=False)
pprint(lda_model.print_topics())
vis = pyLDAvis.gensim.prepare(lda_model, corpus, dicOfTweets)

import re
import numpy as np
import pandas as pd
from pprint import pprint

# Gensim
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess, lemmatize
from gensim.models import CoherenceModel

# spacy for lemmatization
import spacy

import pyLDAvis
import pyLDAvis.gensim
import matplotlib.pyplot as plt

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

# NTLK Stop words
from stop_words import get_stop_words
en_stop = get_stop_words('en')


def sent_to_words(sentences):
    for sentence in sentences:
        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True)) #deacc removes any punctuation


def remove_stopwords(texts):
    return [[word for word in simple_preprocess(str(doc)) if word not in en_stop] for doc in texts]

def make_bigrams(texts):
    return [bigram_mod[doc] for doc in texts]

def make_trigrams(texts):
    return [trigram_mod[bigram_mod[doc]] for doc in texts]

def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    # lemmatization groups together inflicted forms of a verb so it can be analyzed as a single item
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in texts:
        doc = nlp(" ".join(sent))
        texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])
    return texts_out


df = pd.read_json('C:\\Users\\atapia26\\Desktop\\Things for college\\Spring 2020\\Computer Science\\CS 398\\clean_textv3.json')
print(df.text.unique())
print(df.head())

# convert to list
data = df.text.values.tolist()

pprint(data[1])

# remove URLs, @user, and hashtags from the tweet text
data = [' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ", sent).split()) for sent in data]
pprint(data[1])

data_words = list(sent_to_words(data))
sample_data_words = data_words[:10000] # contains only 10000 tweets
print(data_words[:4])

bigram = gensim.models.Phrases(sample_data_words, min_count = 5, threshold = 100)
trigram = gensim.models.Phrases(bigram[sample_data_words], threshold=100)

bigram_mod = gensim.models.phrases.Phraser(bigram)
trigram_mod = gensim.models.phrases.Phraser(trigram)

print(trigram_mod[bigram_mod[sample_data_words[1]]])

data_words_nostops = remove_stopwords(sample_data_words) # remove any unnecessary stop words
data_words_bigrams = make_bigrams(data_words_nostops) # form bigrams

nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])

# lemmatized tweets
data_lemmatized = lemmatization(data_words_bigrams, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])

print(data_lemmatized[1])

# create the dictionary and corpus needed for the topic modeling
dicOfTweets = corpora.Dictionary(data_lemmatized)

tweetTexts = data_lemmatized
# creates the bag of words (term document frequency)
corpus = [dicOfTweets.doc2bow(text) for text in tweetTexts]
# this is to make the dictionary more readable
# corpus = [[(dicOfTweets[id], freq) for id, freq in cp] for cp in corpus]
print(corpus[1])
alphas = np.ones(10)/10 # have 10 topics and I likely believe all these topics are equally likely
lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dicOfTweets, num_topics=10,
                                            random_state=100, update_every=0, chunksize=500,
                                            passes=5, alpha=alphas, per_word_topics=True, distributed=False)
pprint(lda_model.print_topics())
#pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, dicOfTweets)
