# -*- coding: utf-8 -*-
"""
Created on Mon Dec 28 15:37:03 2020

@author: Pablo Otero
"""

import re
import string
import emoji


#HappyEmoticons
emoticons_happy = set([
    ':-)', ':)', ';)', ':o)', ':]', ':3', ':c)', ':>', '=]', '8)', '=)', ':}',
    ':^)', ':-D', ':D', '8-D', '8D', 'x-D', 'xD', 'X-D', 'XD', '=-D', '=D',
    '=-3', '=3', ':-))', ":'-)", ":')", ':*', ':^*', '>:P', ':-P', ':P', 'X-P',
    'x-p', 'xp', 'XP', ':-p', ':p', '=p', ':-b', ':b', '>:)', '>;)', '>:-)',
    '<3'
    ])

# Sad Emoticons
emoticons_sad = set([
    ':L', ':-/', '>:/', ':S', '>:[', ':@', ':-(', ':[', ':-||', '=L', ':<',
    ':-[', ':-<', '=\\', '=/', '>:(', ':(', '>.<', ":'-(", ":'(", ':\\', ':-c',
    ':c', ':{', '>:\\', ';('
    ])

#Emoji patterns
emoji_pattern = re.compile("["
         u"\U0001F600-\U0001F64F"  # emoticons
         u"\U0001F300-\U0001F5FF"  # symbols & pictographs
         u"\U0001F680-\U0001F6FF"  # transport & map symbols
         u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
         u"\U00002702-\U000027B0"
         u"\U000024C2-\U0001F251"
         "]+", flags=re.UNICODE)

#combine sad and happy emoticons
emoticons = emoticons_happy.union(emoticons_sad)


# self defined contractions
def load_dict_contractions():
    
    return {
        "ain't":"is not",
        "amn't":"am not",
        "aren't":"are not",
        "can't":"cannot",
        "'cause":"because",
        "couldn't":"could not",
        "couldn't've":"could not have",
        "could've":"could have",
        "daren't":"dare not",
        "daresn't":"dare not",
        "dasn't":"dare not",
        "didn't":"did not",
        "doesn't":"does not",
        "don't":"do not",
        "e'er":"ever",
        "em":"them",
        "everyone's":"everyone is",
        "finna":"fixing to",
        "gimme":"give me",
        "gonna":"going to",
        "gon't":"go not",
        "gotta":"got to",
        "hadn't":"had not",
        "hasn't":"has not",
        "haven't":"have not",
        "he'd":"he would",
        "he'll":"he will",
        "he's":"he is",
        "he've":"he have",
        "how'd":"how would",
        "how'll":"how will",
        "how're":"how are",
        "how's":"how is",
        "I'd":"I would",
        "I'll":"I will",
        "I'm":"I am",
        "I'm'a":"I am about to",
        "I'm'o":"I am going to",
        "isn't":"is not",
        "it'd":"it would",
        "it'll":"it will",
        "it's":"it is",
        "I've":"I have",
        "kinda":"kind of",
        "let's":"let us",
        "mayn't":"may not",
        "may've":"may have",
        "mightn't":"might not",
        "might've":"might have",
        "mustn't":"must not",
        "mustn't've":"must not have",
        "must've":"must have",
        "needn't":"need not",
        "ne'er":"never",
        "o'":"of",
        "o'er":"over",
        "ol'":"old",
        "oughtn't":"ought not",
        "shalln't":"shall not",
        "shan't":"shall not",
        "she'd":"she would",
        "she'll":"she will",
        "she's":"she is",
        "shouldn't":"should not",
        "shouldn't've":"should not have",
        "should've":"should have",
        "somebody's":"somebody is",
        "someone's":"someone is",
        "something's":"something is",
        "that'd":"that would",
        "that'll":"that will",
        "that're":"that are",
        "that's":"that is",
        "there'd":"there would",
        "there'll":"there will",
        "there're":"there are",
        "there's":"there is",
        "these're":"these are",
        "they'd":"they would",
        "they'll":"they will",
        "they're":"they are",
        "they've":"they have",
        "this's":"this is",
        "those're":"those are",
        "'tis":"it is",
        "'twas":"it was",
        "wanna":"want to",
        "wasn't":"was not",
        "we'd":"we would",
        "we'd've":"we would have",
        "we'll":"we will",
        "we're":"we are",
        "weren't":"were not",
        "we've":"we have",
        "what'd":"what did",
        "what'll":"what will",
        "what're":"what are",
        "what's":"what is",
        "what've":"what have",
        "when's":"when is",
        "where'd":"where did",
        "where're":"where are",
        "where's":"where is",
        "where've":"where have",
        "which's":"which is",
        "who'd":"who would",
        "who'd've":"who would have",
        "who'll":"who will",
        "who're":"who are",
        "who's":"who is",
        "who've":"who have",
        "why'd":"why did",
        "why're":"why are",
        "why's":"why is",
        "won't":"will not",
        "wouldn't":"would not",
        "would've":"would have",
        "y'all":"you all",
        "you'd":"you would",
        "you'll":"you will",
        "you're":"you are",
        "you've":"you have",
        "Whatcha":"What are you",
        "luv":"love",
        "sux":"sucks"
        }


# Fix classic tweet lingo
def fix_lingo(tweet):
    
    tweet = re.sub(r'\bthats\b', 'that is', tweet)
    tweet = re.sub(r'\bive\b', 'i have', tweet)
    tweet = re.sub(r'\bim\b', 'i am', tweet)
    tweet = re.sub(r'\bya\b', 'yeah', tweet)
    tweet = re.sub(r'\bcant\b', 'can not', tweet)
    tweet = re.sub(r'\bwont\b', 'will not', tweet)
    tweet = re.sub(r'\bid\b', 'i would', tweet)
    tweet = re.sub(r'wtf', 'what the fuck', tweet)
    tweet = re.sub(r'\bwth\b', 'what the hell', tweet)
    tweet = re.sub(r'\br\b', 'are', tweet)
    tweet = re.sub(r'\bu\b', 'you', tweet)
    tweet = re.sub(r'\bk\b', 'OK', tweet)
    tweet = re.sub(r'\bsux\b', 'sucks', tweet)
    tweet = re.sub(r'\bno+\b', 'no', tweet)
    tweet = re.sub(r'\bcoo+\b', 'cool', tweet) 
    
    return tweet


#Gets the text, removes links, hashtags, mentions, media, and symbols.
def get_text_cleaned(status):
    """
    status is the object obtained after calling the Twitter API
    """
    
    #text = status['text']
    text = status['full_text']
    
    slices = []
    #Strip out the urls.
    if 'urls' in status['entities']:
        for url in status['entities']['urls']:
            slices += [{'start': url['indices'][0], 'stop': url['indices'][1]}]
    
    #Strip out the hashtags.
    if 'hashtags' in status['entities']:
        for tag in status['entities']['hashtags']:
            slices += [{'start': tag['indices'][0], 'stop': tag['indices'][1]}]
    
    #Strip out the user mentions.
    if 'user_mentions' in status['entities']:
        for men in status['entities']['user_mentions']:
            slices += [{'start': men['indices'][0], 'stop': men['indices'][1]}]
    
    #Strip out the media.
    if 'media' in status['entities']:
        for med in status['entities']['media']:
            slices += [{'start': med['indices'][0], 'stop': med['indices'][1]}]
    
    #Strip out the symbols.
    if 'symbols' in status['entities']:
        for sym in status['entities']['symbols']:
            slices += [{'start': sym['indices'][0], 'stop': sym['indices'][1]}]
    
    # Sort the slices from highest start to lowest.
    slices = sorted(slices, key=lambda x: -x['start'])
    
    #No offsets, since we're sorted from highest to lowest.
    for s in slices:
        text = text[:s['start']] + text[s['stop']:]
        
    return text


#method full_clean_tweets()
def full_clean_tweets(tweet):
    stop_words = set(stopwords.words('english'))
    stop_words.update(stopwords.words('spanish'))
    word_tokens = word_tokenize(tweet)

    #after tweepy preprocessing the colon left remain after removing mentions
    #or RT sign in the beginning of the tweet
    tweet = re.sub(r':', '', tweet)
    tweet = re.sub(r'‚Ä¶', '', tweet)
    #replace consecutive non-ASCII characters with a space
    tweet = re.sub(r'[^\x00-\x7F]+',' ', tweet)

    #remove emojis from tweet
    tweet = emoji_pattern.sub(r'', tweet)

    #filter using NLTK library append it to a string
    filtered_tweet = [w for w in word_tokens if not w in stop_words]
    filtered_tweet = []

    #looping through conditions
    for w in word_tokens:
        #check tokens against stop words , emoticons and punctuations
        if w not in stop_words and w not in emoticons and w not in string.punctuation:
            filtered_tweet.append(w)
    return ' '.join(filtered_tweet)


def get_text_sanitized(status):    
    #or RT sign in the beginning of the tweet
    
    tweet = get_text_cleaned(status)
    
    tweet = re.sub(r'RT:', '', tweet)
    tweet = re.sub(r'@', '', tweet)
    
    #replace consecutive non-ASCII characters with a space
    ##note that emojis and emoticons are lost
    #tweet = re.sub(r'[^\x00-\x7F]+',' ', tweet)

    #normalize case
    tweet = tweet.lower()
    
    return tweet


def extract_emojis(str):
    return ''.join(c for c in str if c in emoji.UNICODE_EMOJI)


def strip_non_ascii(string):
    ''' Returns the string without non ASCII characters'''
    stripped = (c for c in string if 0 < ord(c) < 127)
    return ''.join(stripped)





c46db94c80d0139471b739b3cffef6afe95184a7
<mask token>


def load_dict_contractions():
    return {"ain't": 'is not', "amn't": 'am not', "aren't": 'are not',
        "can't": 'cannot', "'cause": 'because', "couldn't": 'could not',
        "couldn't've": 'could not have', "could've": 'could have',
        "daren't": 'dare not', "daresn't": 'dare not', "dasn't": 'dare not',
        "didn't": 'did not', "doesn't": 'does not', "don't": 'do not',
        "e'er": 'ever', 'em': 'them', "everyone's": 'everyone is', 'finna':
        'fixing to', 'gimme': 'give me', 'gonna': 'going to', "gon't":
        'go not', 'gotta': 'got to', "hadn't": 'had not', "hasn't":
        'has not', "haven't": 'have not', "he'd": 'he would', "he'll":
        'he will', "he's": 'he is', "he've": 'he have', "how'd":
        'how would', "how'll": 'how will', "how're": 'how are', "how's":
        'how is', "I'd": 'I would', "I'll": 'I will', "I'm": 'I am',
        "I'm'a": 'I am about to', "I'm'o": 'I am going to', "isn't":
        'is not', "it'd": 'it would', "it'll": 'it will', "it's": 'it is',
        "I've": 'I have', 'kinda': 'kind of', "let's": 'let us', "mayn't":
        'may not', "may've": 'may have', "mightn't": 'might not',
        "might've": 'might have', "mustn't": 'must not', "mustn't've":
        'must not have', "must've": 'must have', "needn't": 'need not',
        "ne'er": 'never', "o'": 'of', "o'er": 'over', "ol'": 'old',
        "oughtn't": 'ought not', "shalln't": 'shall not', "shan't":
        'shall not', "she'd": 'she would', "she'll": 'she will', "she's":
        'she is', "shouldn't": 'should not', "shouldn't've":
        'should not have', "should've": 'should have', "somebody's":
        'somebody is', "someone's": 'someone is', "something's":
        'something is', "that'd": 'that would', "that'll": 'that will',
        "that're": 'that are', "that's": 'that is', "there'd":
        'there would', "there'll": 'there will', "there're": 'there are',
        "there's": 'there is', "these're": 'these are', "they'd":
        'they would', "they'll": 'they will', "they're": 'they are',
        "they've": 'they have', "this's": 'this is', "those're":
        'those are', "'tis": 'it is', "'twas": 'it was', 'wanna': 'want to',
        "wasn't": 'was not', "we'd": 'we would', "we'd've": 'we would have',
        "we'll": 'we will', "we're": 'we are', "weren't": 'were not',
        "we've": 'we have', "what'd": 'what did', "what'll": 'what will',
        "what're": 'what are', "what's": 'what is', "what've": 'what have',
        "when's": 'when is', "where'd": 'where did', "where're":
        'where are', "where's": 'where is', "where've": 'where have',
        "which's": 'which is', "who'd": 'who would', "who'd've":
        'who would have', "who'll": 'who will', "who're": 'who are',
        "who's": 'who is', "who've": 'who have', "why'd": 'why did',
        "why're": 'why are', "why's": 'why is', "won't": 'will not',
        "wouldn't": 'would not', "would've": 'would have', "y'all":
        'you all', "you'd": 'you would', "you'll": 'you will', "you're":
        'you are', "you've": 'you have', 'Whatcha': 'What are you', 'luv':
        'love', 'sux': 'sucks'}


<mask token>


def extract_emojis(str):
    return ''.join(c for c in str if c in emoji.UNICODE_EMOJI)


def strip_non_ascii(string):
    """ Returns the string without non ASCII characters"""
    stripped = (c for c in string if 0 < ord(c) < 127)
    return ''.join(stripped)

<mask token>


def load_dict_contractions():
    return {"ain't": 'is not', "amn't": 'am not', "aren't": 'are not',
        "can't": 'cannot', "'cause": 'because', "couldn't": 'could not',
        "couldn't've": 'could not have', "could've": 'could have',
        "daren't": 'dare not', "daresn't": 'dare not', "dasn't": 'dare not',
        "didn't": 'did not', "doesn't": 'does not', "don't": 'do not',
        "e'er": 'ever', 'em': 'them', "everyone's": 'everyone is', 'finna':
        'fixing to', 'gimme': 'give me', 'gonna': 'going to', "gon't":
        'go not', 'gotta': 'got to', "hadn't": 'had not', "hasn't":
        'has not', "haven't": 'have not', "he'd": 'he would', "he'll":
        'he will', "he's": 'he is', "he've": 'he have', "how'd":
        'how would', "how'll": 'how will', "how're": 'how are', "how's":
        'how is', "I'd": 'I would', "I'll": 'I will', "I'm": 'I am',
        "I'm'a": 'I am about to', "I'm'o": 'I am going to', "isn't":
        'is not', "it'd": 'it would', "it'll": 'it will', "it's": 'it is',
        "I've": 'I have', 'kinda': 'kind of', "let's": 'let us', "mayn't":
        'may not', "may've": 'may have', "mightn't": 'might not',
        "might've": 'might have', "mustn't": 'must not', "mustn't've":
        'must not have', "must've": 'must have', "needn't": 'need not',
        "ne'er": 'never', "o'": 'of', "o'er": 'over', "ol'": 'old',
        "oughtn't": 'ought not', "shalln't": 'shall not', "shan't":
        'shall not', "she'd": 'she would', "she'll": 'she will', "she's":
        'she is', "shouldn't": 'should not', "shouldn't've":
        'should not have', "should've": 'should have', "somebody's":
        'somebody is', "someone's": 'someone is', "something's":
        'something is', "that'd": 'that would', "that'll": 'that will',
        "that're": 'that are', "that's": 'that is', "there'd":
        'there would', "there'll": 'there will', "there're": 'there are',
        "there's": 'there is', "these're": 'these are', "they'd":
        'they would', "they'll": 'they will', "they're": 'they are',
        "they've": 'they have', "this's": 'this is', "those're":
        'those are', "'tis": 'it is', "'twas": 'it was', 'wanna': 'want to',
        "wasn't": 'was not', "we'd": 'we would', "we'd've": 'we would have',
        "we'll": 'we will', "we're": 'we are', "weren't": 'were not',
        "we've": 'we have', "what'd": 'what did', "what'll": 'what will',
        "what're": 'what are', "what's": 'what is', "what've": 'what have',
        "when's": 'when is', "where'd": 'where did', "where're":
        'where are', "where's": 'where is', "where've": 'where have',
        "which's": 'which is', "who'd": 'who would', "who'd've":
        'who would have', "who'll": 'who will', "who're": 'who are',
        "who's": 'who is', "who've": 'who have', "why'd": 'why did',
        "why're": 'why are', "why's": 'why is', "won't": 'will not',
        "wouldn't": 'would not', "would've": 'would have', "y'all":
        'you all', "you'd": 'you would', "you'll": 'you will', "you're":
        'you are', "you've": 'you have', 'Whatcha': 'What are you', 'luv':
        'love', 'sux': 'sucks'}


def fix_lingo(tweet):
    tweet = re.sub('\\bthats\\b', 'that is', tweet)
    tweet = re.sub('\\bive\\b', 'i have', tweet)
    tweet = re.sub('\\bim\\b', 'i am', tweet)
    tweet = re.sub('\\bya\\b', 'yeah', tweet)
    tweet = re.sub('\\bcant\\b', 'can not', tweet)
    tweet = re.sub('\\bwont\\b', 'will not', tweet)
    tweet = re.sub('\\bid\\b', 'i would', tweet)
    tweet = re.sub('wtf', 'what the fuck', tweet)
    tweet = re.sub('\\bwth\\b', 'what the hell', tweet)
    tweet = re.sub('\\br\\b', 'are', tweet)
    tweet = re.sub('\\bu\\b', 'you', tweet)
    tweet = re.sub('\\bk\\b', 'OK', tweet)
    tweet = re.sub('\\bsux\\b', 'sucks', tweet)
    tweet = re.sub('\\bno+\\b', 'no', tweet)
    tweet = re.sub('\\bcoo+\\b', 'cool', tweet)
    return tweet


def get_text_cleaned(status):
    """
    status is the object obtained after calling the Twitter API
    """
    text = status['full_text']
    slices = []
    if 'urls' in status['entities']:
        for url in status['entities']['urls']:
            slices += [{'start': url['indices'][0], 'stop': url['indices'][1]}]
    if 'hashtags' in status['entities']:
        for tag in status['entities']['hashtags']:
            slices += [{'start': tag['indices'][0], 'stop': tag['indices'][1]}]
    if 'user_mentions' in status['entities']:
        for men in status['entities']['user_mentions']:
            slices += [{'start': men['indices'][0], 'stop': men['indices'][1]}]
    if 'media' in status['entities']:
        for med in status['entities']['media']:
            slices += [{'start': med['indices'][0], 'stop': med['indices'][1]}]
    if 'symbols' in status['entities']:
        for sym in status['entities']['symbols']:
            slices += [{'start': sym['indices'][0], 'stop': sym['indices'][1]}]
    slices = sorted(slices, key=lambda x: -x['start'])
    for s in slices:
        text = text[:s['start']] + text[s['stop']:]
    return text


def full_clean_tweets(tweet):
    stop_words = set(stopwords.words('english'))
    stop_words.update(stopwords.words('spanish'))
    word_tokens = word_tokenize(tweet)
    tweet = re.sub(':', '', tweet)
    tweet = re.sub('‚Ä¶', '', tweet)
    tweet = re.sub('[^\\x00-\\x7F]+', ' ', tweet)
    tweet = emoji_pattern.sub('', tweet)
    filtered_tweet = [w for w in word_tokens if not w in stop_words]
    filtered_tweet = []
    for w in word_tokens:
        if (w not in stop_words and w not in emoticons and w not in string.
            punctuation):
            filtered_tweet.append(w)
    return ' '.join(filtered_tweet)


def get_text_sanitized(status):
    tweet = get_text_cleaned(status)
    tweet = re.sub('RT:', '', tweet)
    tweet = re.sub('@', '', tweet)
    tweet = tweet.lower()
    return tweet


def extract_emojis(str):
    return ''.join(c for c in str if c in emoji.UNICODE_EMOJI)


def strip_non_ascii(string):
    """ Returns the string without non ASCII characters"""
    stripped = (c for c in string if 0 < ord(c) < 127)
    return ''.join(stripped)

<mask token>
emoticons_happy = set([':-)', ':)', ';)', ':o)', ':]', ':3', ':c)', ':>',
    '=]', '8)', '=)', ':}', ':^)', ':-D', ':D', '8-D', '8D', 'x-D', 'xD',
    'X-D', 'XD', '=-D', '=D', '=-3', '=3', ':-))', ":'-)", ":')", ':*',
    ':^*', '>:P', ':-P', ':P', 'X-P', 'x-p', 'xp', 'XP', ':-p', ':p', '=p',
    ':-b', ':b', '>:)', '>;)', '>:-)', '<3'])
emoticons_sad = set([':L', ':-/', '>:/', ':S', '>:[', ':@', ':-(', ':[',
    ':-||', '=L', ':<', ':-[', ':-<', '=\\', '=/', '>:(', ':(', '>.<',
    ":'-(", ":'(", ':\\', ':-c', ':c', ':{', '>:\\', ';('])
emoji_pattern = re.compile('[😀-🙏🌀-🗿🚀-\U0001f6ff\U0001f1e0-🇿✂-➰Ⓜ-🉑]+', flags
    =re.UNICODE)
emoticons = emoticons_happy.union(emoticons_sad)


def load_dict_contractions():
    return {"ain't": 'is not', "amn't": 'am not', "aren't": 'are not',
        "can't": 'cannot', "'cause": 'because', "couldn't": 'could not',
        "couldn't've": 'could not have', "could've": 'could have',
        "daren't": 'dare not', "daresn't": 'dare not', "dasn't": 'dare not',
        "didn't": 'did not', "doesn't": 'does not', "don't": 'do not',
        "e'er": 'ever', 'em': 'them', "everyone's": 'everyone is', 'finna':
        'fixing to', 'gimme': 'give me', 'gonna': 'going to', "gon't":
        'go not', 'gotta': 'got to', "hadn't": 'had not', "hasn't":
        'has not', "haven't": 'have not', "he'd": 'he would', "he'll":
        'he will', "he's": 'he is', "he've": 'he have', "how'd":
        'how would', "how'll": 'how will', "how're": 'how are', "how's":
        'how is', "I'd": 'I would', "I'll": 'I will', "I'm": 'I am',
        "I'm'a": 'I am about to', "I'm'o": 'I am going to', "isn't":
        'is not', "it'd": 'it would', "it'll": 'it will', "it's": 'it is',
        "I've": 'I have', 'kinda': 'kind of', "let's": 'let us', "mayn't":
        'may not', "may've": 'may have', "mightn't": 'might not',
        "might've": 'might have', "mustn't": 'must not', "mustn't've":
        'must not have', "must've": 'must have', "needn't": 'need not',
        "ne'er": 'never', "o'": 'of', "o'er": 'over', "ol'": 'old',
        "oughtn't": 'ought not', "shalln't": 'shall not', "shan't":
        'shall not', "she'd": 'she would', "she'll": 'she will', "she's":
        'she is', "shouldn't": 'should not', "shouldn't've":
        'should not have', "should've": 'should have', "somebody's":
        'somebody is', "someone's": 'someone is', "something's":
        'something is', "that'd": 'that would', "that'll": 'that will',
        "that're": 'that are', "that's": 'that is', "there'd":
        'there would', "there'll": 'there will', "there're": 'there are',
        "there's": 'there is', "these're": 'these are', "they'd":
        'they would', "they'll": 'they will', "they're": 'they are',
        "they've": 'they have', "this's": 'this is', "those're":
        'those are', "'tis": 'it is', "'twas": 'it was', 'wanna': 'want to',
        "wasn't": 'was not', "we'd": 'we would', "we'd've": 'we would have',
        "we'll": 'we will', "we're": 'we are', "weren't": 'were not',
        "we've": 'we have', "what'd": 'what did', "what'll": 'what will',
        "what're": 'what are', "what's": 'what is', "what've": 'what have',
        "when's": 'when is', "where'd": 'where did', "where're":
        'where are', "where's": 'where is', "where've": 'where have',
        "which's": 'which is', "who'd": 'who would', "who'd've":
        'who would have', "who'll": 'who will', "who're": 'who are',
        "who's": 'who is', "who've": 'who have', "why'd": 'why did',
        "why're": 'why are', "why's": 'why is', "won't": 'will not',
        "wouldn't": 'would not', "would've": 'would have', "y'all":
        'you all', "you'd": 'you would', "you'll": 'you will', "you're":
        'you are', "you've": 'you have', 'Whatcha': 'What are you', 'luv':
        'love', 'sux': 'sucks'}


def fix_lingo(tweet):
    tweet = re.sub('\\bthats\\b', 'that is', tweet)
    tweet = re.sub('\\bive\\b', 'i have', tweet)
    tweet = re.sub('\\bim\\b', 'i am', tweet)
    tweet = re.sub('\\bya\\b', 'yeah', tweet)
    tweet = re.sub('\\bcant\\b', 'can not', tweet)
    tweet = re.sub('\\bwont\\b', 'will not', tweet)
    tweet = re.sub('\\bid\\b', 'i would', tweet)
    tweet = re.sub('wtf', 'what the fuck', tweet)
    tweet = re.sub('\\bwth\\b', 'what the hell', tweet)
    tweet = re.sub('\\br\\b', 'are', tweet)
    tweet = re.sub('\\bu\\b', 'you', tweet)
    tweet = re.sub('\\bk\\b', 'OK', tweet)
    tweet = re.sub('\\bsux\\b', 'sucks', tweet)
    tweet = re.sub('\\bno+\\b', 'no', tweet)
    tweet = re.sub('\\bcoo+\\b', 'cool', tweet)
    return tweet


def get_text_cleaned(status):
    """
    status is the object obtained after calling the Twitter API
    """
    text = status['full_text']
    slices = []
    if 'urls' in status['entities']:
        for url in status['entities']['urls']:
            slices += [{'start': url['indices'][0], 'stop': url['indices'][1]}]
    if 'hashtags' in status['entities']:
        for tag in status['entities']['hashtags']:
            slices += [{'start': tag['indices'][0], 'stop': tag['indices'][1]}]
    if 'user_mentions' in status['entities']:
        for men in status['entities']['user_mentions']:
            slices += [{'start': men['indices'][0], 'stop': men['indices'][1]}]
    if 'media' in status['entities']:
        for med in status['entities']['media']:
            slices += [{'start': med['indices'][0], 'stop': med['indices'][1]}]
    if 'symbols' in status['entities']:
        for sym in status['entities']['symbols']:
            slices += [{'start': sym['indices'][0], 'stop': sym['indices'][1]}]
    slices = sorted(slices, key=lambda x: -x['start'])
    for s in slices:
        text = text[:s['start']] + text[s['stop']:]
    return text


def full_clean_tweets(tweet):
    stop_words = set(stopwords.words('english'))
    stop_words.update(stopwords.words('spanish'))
    word_tokens = word_tokenize(tweet)
    tweet = re.sub(':', '', tweet)
    tweet = re.sub('‚Ä¶', '', tweet)
    tweet = re.sub('[^\\x00-\\x7F]+', ' ', tweet)
    tweet = emoji_pattern.sub('', tweet)
    filtered_tweet = [w for w in word_tokens if not w in stop_words]
    filtered_tweet = []
    for w in word_tokens:
        if (w not in stop_words and w not in emoticons and w not in string.
            punctuation):
            filtered_tweet.append(w)
    return ' '.join(filtered_tweet)


def get_text_sanitized(status):
    tweet = get_text_cleaned(status)
    tweet = re.sub('RT:', '', tweet)
    tweet = re.sub('@', '', tweet)
    tweet = tweet.lower()
    return tweet


def extract_emojis(str):
    return ''.join(c for c in str if c in emoji.UNICODE_EMOJI)


def strip_non_ascii(string):
    """ Returns the string without non ASCII characters"""
    stripped = (c for c in string if 0 < ord(c) < 127)
    return ''.join(stripped)

<mask token>
import re
import string
import emoji
emoticons_happy = set([':-)', ':)', ';)', ':o)', ':]', ':3', ':c)', ':>',
    '=]', '8)', '=)', ':}', ':^)', ':-D', ':D', '8-D', '8D', 'x-D', 'xD',
    'X-D', 'XD', '=-D', '=D', '=-3', '=3', ':-))', ":'-)", ":')", ':*',
    ':^*', '>:P', ':-P', ':P', 'X-P', 'x-p', 'xp', 'XP', ':-p', ':p', '=p',
    ':-b', ':b', '>:)', '>;)', '>:-)', '<3'])
emoticons_sad = set([':L', ':-/', '>:/', ':S', '>:[', ':@', ':-(', ':[',
    ':-||', '=L', ':<', ':-[', ':-<', '=\\', '=/', '>:(', ':(', '>.<',
    ":'-(", ":'(", ':\\', ':-c', ':c', ':{', '>:\\', ';('])
emoji_pattern = re.compile('[😀-🙏🌀-🗿🚀-\U0001f6ff\U0001f1e0-🇿✂-➰Ⓜ-🉑]+', flags
    =re.UNICODE)
emoticons = emoticons_happy.union(emoticons_sad)


def load_dict_contractions():
    return {"ain't": 'is not', "amn't": 'am not', "aren't": 'are not',
        "can't": 'cannot', "'cause": 'because', "couldn't": 'could not',
        "couldn't've": 'could not have', "could've": 'could have',
        "daren't": 'dare not', "daresn't": 'dare not', "dasn't": 'dare not',
        "didn't": 'did not', "doesn't": 'does not', "don't": 'do not',
        "e'er": 'ever', 'em': 'them', "everyone's": 'everyone is', 'finna':
        'fixing to', 'gimme': 'give me', 'gonna': 'going to', "gon't":
        'go not', 'gotta': 'got to', "hadn't": 'had not', "hasn't":
        'has not', "haven't": 'have not', "he'd": 'he would', "he'll":
        'he will', "he's": 'he is', "he've": 'he have', "how'd":
        'how would', "how'll": 'how will', "how're": 'how are', "how's":
        'how is', "I'd": 'I would', "I'll": 'I will', "I'm": 'I am',
        "I'm'a": 'I am about to', "I'm'o": 'I am going to', "isn't":
        'is not', "it'd": 'it would', "it'll": 'it will', "it's": 'it is',
        "I've": 'I have', 'kinda': 'kind of', "let's": 'let us', "mayn't":
        'may not', "may've": 'may have', "mightn't": 'might not',
        "might've": 'might have', "mustn't": 'must not', "mustn't've":
        'must not have', "must've": 'must have', "needn't": 'need not',
        "ne'er": 'never', "o'": 'of', "o'er": 'over', "ol'": 'old',
        "oughtn't": 'ought not', "shalln't": 'shall not', "shan't":
        'shall not', "she'd": 'she would', "she'll": 'she will', "she's":
        'she is', "shouldn't": 'should not', "shouldn't've":
        'should not have', "should've": 'should have', "somebody's":
        'somebody is', "someone's": 'someone is', "something's":
        'something is', "that'd": 'that would', "that'll": 'that will',
        "that're": 'that are', "that's": 'that is', "there'd":
        'there would', "there'll": 'there will', "there're": 'there are',
        "there's": 'there is', "these're": 'these are', "they'd":
        'they would', "they'll": 'they will', "they're": 'they are',
        "they've": 'they have', "this's": 'this is', "those're":
        'those are', "'tis": 'it is', "'twas": 'it was', 'wanna': 'want to',
        "wasn't": 'was not', "we'd": 'we would', "we'd've": 'we would have',
        "we'll": 'we will', "we're": 'we are', "weren't": 'were not',
        "we've": 'we have', "what'd": 'what did', "what'll": 'what will',
        "what're": 'what are', "what's": 'what is', "what've": 'what have',
        "when's": 'when is', "where'd": 'where did', "where're":
        'where are', "where's": 'where is', "where've": 'where have',
        "which's": 'which is', "who'd": 'who would', "who'd've":
        'who would have', "who'll": 'who will', "who're": 'who are',
        "who's": 'who is', "who've": 'who have', "why'd": 'why did',
        "why're": 'why are', "why's": 'why is', "won't": 'will not',
        "wouldn't": 'would not', "would've": 'would have', "y'all":
        'you all', "you'd": 'you would', "you'll": 'you will', "you're":
        'you are', "you've": 'you have', 'Whatcha': 'What are you', 'luv':
        'love', 'sux': 'sucks'}


def fix_lingo(tweet):
    tweet = re.sub('\\bthats\\b', 'that is', tweet)
    tweet = re.sub('\\bive\\b', 'i have', tweet)
    tweet = re.sub('\\bim\\b', 'i am', tweet)
    tweet = re.sub('\\bya\\b', 'yeah', tweet)
    tweet = re.sub('\\bcant\\b', 'can not', tweet)
    tweet = re.sub('\\bwont\\b', 'will not', tweet)
    tweet = re.sub('\\bid\\b', 'i would', tweet)
    tweet = re.sub('wtf', 'what the fuck', tweet)
    tweet = re.sub('\\bwth\\b', 'what the hell', tweet)
    tweet = re.sub('\\br\\b', 'are', tweet)
    tweet = re.sub('\\bu\\b', 'you', tweet)
    tweet = re.sub('\\bk\\b', 'OK', tweet)
    tweet = re.sub('\\bsux\\b', 'sucks', tweet)
    tweet = re.sub('\\bno+\\b', 'no', tweet)
    tweet = re.sub('\\bcoo+\\b', 'cool', tweet)
    return tweet


def get_text_cleaned(status):
    """
    status is the object obtained after calling the Twitter API
    """
    text = status['full_text']
    slices = []
    if 'urls' in status['entities']:
        for url in status['entities']['urls']:
            slices += [{'start': url['indices'][0], 'stop': url['indices'][1]}]
    if 'hashtags' in status['entities']:
        for tag in status['entities']['hashtags']:
            slices += [{'start': tag['indices'][0], 'stop': tag['indices'][1]}]
    if 'user_mentions' in status['entities']:
        for men in status['entities']['user_mentions']:
            slices += [{'start': men['indices'][0], 'stop': men['indices'][1]}]
    if 'media' in status['entities']:
        for med in status['entities']['media']:
            slices += [{'start': med['indices'][0], 'stop': med['indices'][1]}]
    if 'symbols' in status['entities']:
        for sym in status['entities']['symbols']:
            slices += [{'start': sym['indices'][0], 'stop': sym['indices'][1]}]
    slices = sorted(slices, key=lambda x: -x['start'])
    for s in slices:
        text = text[:s['start']] + text[s['stop']:]
    return text


def full_clean_tweets(tweet):
    stop_words = set(stopwords.words('english'))
    stop_words.update(stopwords.words('spanish'))
    word_tokens = word_tokenize(tweet)
    tweet = re.sub(':', '', tweet)
    tweet = re.sub('‚Ä¶', '', tweet)
    tweet = re.sub('[^\\x00-\\x7F]+', ' ', tweet)
    tweet = emoji_pattern.sub('', tweet)
    filtered_tweet = [w for w in word_tokens if not w in stop_words]
    filtered_tweet = []
    for w in word_tokens:
        if (w not in stop_words and w not in emoticons and w not in string.
            punctuation):
            filtered_tweet.append(w)
    return ' '.join(filtered_tweet)


def get_text_sanitized(status):
    tweet = get_text_cleaned(status)
    tweet = re.sub('RT:', '', tweet)
    tweet = re.sub('@', '', tweet)
    tweet = tweet.lower()
    return tweet


def extract_emojis(str):
    return ''.join(c for c in str if c in emoji.UNICODE_EMOJI)


def strip_non_ascii(string):
    """ Returns the string without non ASCII characters"""
    stripped = (c for c in string if 0 < ord(c) < 127)
    return ''.join(stripped)

# -*- coding: utf-8 -*-
"""
Created on Mon Dec 28 15:37:03 2020

@author: Pablo Otero
"""

import re
import string
import emoji


#HappyEmoticons
emoticons_happy = set([
    ':-)', ':)', ';)', ':o)', ':]', ':3', ':c)', ':>', '=]', '8)', '=)', ':}',
    ':^)', ':-D', ':D', '8-D', '8D', 'x-D', 'xD', 'X-D', 'XD', '=-D', '=D',
    '=-3', '=3', ':-))', ":'-)", ":')", ':*', ':^*', '>:P', ':-P', ':P', 'X-P',
    'x-p', 'xp', 'XP', ':-p', ':p', '=p', ':-b', ':b', '>:)', '>;)', '>:-)',
    '<3'
    ])

# Sad Emoticons
emoticons_sad = set([
    ':L', ':-/', '>:/', ':S', '>:[', ':@', ':-(', ':[', ':-||', '=L', ':<',
    ':-[', ':-<', '=\\', '=/', '>:(', ':(', '>.<', ":'-(", ":'(", ':\\', ':-c',
    ':c', ':{', '>:\\', ';('
    ])

#Emoji patterns
emoji_pattern = re.compile("["
         u"\U0001F600-\U0001F64F"  # emoticons
         u"\U0001F300-\U0001F5FF"  # symbols & pictographs
         u"\U0001F680-\U0001F6FF"  # transport & map symbols
         u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
         u"\U00002702-\U000027B0"
         u"\U000024C2-\U0001F251"
         "]+", flags=re.UNICODE)

#combine sad and happy emoticons
emoticons = emoticons_happy.union(emoticons_sad)


# self defined contractions
def load_dict_contractions():
    
    return {
        "ain't":"is not",
        "amn't":"am not",
        "aren't":"are not",
        "can't":"cannot",
        "'cause":"because",
        "couldn't":"could not",
        "couldn't've":"could not have",
        "could've":"could have",
        "daren't":"dare not",
        "daresn't":"dare not",
        "dasn't":"dare not",
        "didn't":"did not",
        "doesn't":"does not",
        "don't":"do not",
        "e'er":"ever",
        "em":"them",
        "everyone's":"everyone is",
        "finna":"fixing to",
        "gimme":"give me",
        "gonna":"going to",
        "gon't":"go not",
        "gotta":"got to",
        "hadn't":"had not",
        "hasn't":"has not",
        "haven't":"have not",
        "he'd":"he would",
        "he'll":"he will",
        "he's":"he is",
        "he've":"he have",
        "how'd":"how would",
        "how'll":"how will",
        "how're":"how are",
        "how's":"how is",
        "I'd":"I would",
        "I'll":"I will",
        "I'm":"I am",
        "I'm'a":"I am about to",
        "I'm'o":"I am going to",
        "isn't":"is not",
        "it'd":"it would",
        "it'll":"it will",
        "it's":"it is",
        "I've":"I have",
        "kinda":"kind of",
        "let's":"let us",
        "mayn't":"may not",
        "may've":"may have",
        "mightn't":"might not",
        "might've":"might have",
        "mustn't":"must not",
        "mustn't've":"must not have",
        "must've":"must have",
        "needn't":"need not",
        "ne'er":"never",
        "o'":"of",
        "o'er":"over",
        "ol'":"old",
        "oughtn't":"ought not",
        "shalln't":"shall not",
        "shan't":"shall not",
        "she'd":"she would",
        "she'll":"she will",
        "she's":"she is",
        "shouldn't":"should not",
        "shouldn't've":"should not have",
        "should've":"should have",
        "somebody's":"somebody is",
        "someone's":"someone is",
        "something's":"something is",
        "that'd":"that would",
        "that'll":"that will",
        "that're":"that are",
        "that's":"that is",
        "there'd":"there would",
        "there'll":"there will",
        "there're":"there are",
        "there's":"there is",
        "these're":"these are",
        "they'd":"they would",
        "they'll":"they will",
        "they're":"they are",
        "they've":"they have",
        "this's":"this is",
        "those're":"those are",
        "'tis":"it is",
        "'twas":"it was",
        "wanna":"want to",
        "wasn't":"was not",
        "we'd":"we would",
        "we'd've":"we would have",
        "we'll":"we will",
        "we're":"we are",
        "weren't":"were not",
        "we've":"we have",
        "what'd":"what did",
        "what'll":"what will",
        "what're":"what are",
        "what's":"what is",
        "what've":"what have",
        "when's":"when is",
        "where'd":"where did",
        "where're":"where are",
        "where's":"where is",
        "where've":"where have",
        "which's":"which is",
        "who'd":"who would",
        "who'd've":"who would have",
        "who'll":"who will",
        "who're":"who are",
        "who's":"who is",
        "who've":"who have",
        "why'd":"why did",
        "why're":"why are",
        "why's":"why is",
        "won't":"will not",
        "wouldn't":"would not",
        "would've":"would have",
        "y'all":"you all",
        "you'd":"you would",
        "you'll":"you will",
        "you're":"you are",
        "you've":"you have",
        "Whatcha":"What are you",
        "luv":"love",
        "sux":"sucks"
        }


# Fix classic tweet lingo
def fix_lingo(tweet):
    
    tweet = re.sub(r'\bthats\b', 'that is', tweet)
    tweet = re.sub(r'\bive\b', 'i have', tweet)
    tweet = re.sub(r'\bim\b', 'i am', tweet)
    tweet = re.sub(r'\bya\b', 'yeah', tweet)
    tweet = re.sub(r'\bcant\b', 'can not', tweet)
    tweet = re.sub(r'\bwont\b', 'will not', tweet)
    tweet = re.sub(r'\bid\b', 'i would', tweet)
    tweet = re.sub(r'wtf', 'what the fuck', tweet)
    tweet = re.sub(r'\bwth\b', 'what the hell', tweet)
    tweet = re.sub(r'\br\b', 'are', tweet)
    tweet = re.sub(r'\bu\b', 'you', tweet)
    tweet = re.sub(r'\bk\b', 'OK', tweet)
    tweet = re.sub(r'\bsux\b', 'sucks', tweet)
    tweet = re.sub(r'\bno+\b', 'no', tweet)
    tweet = re.sub(r'\bcoo+\b', 'cool', tweet) 
    
    return tweet


#Gets the text, removes links, hashtags, mentions, media, and symbols.
def get_text_cleaned(status):
    """
    status is the object obtained after calling the Twitter API
    """
    
    #text = status['text']
    text = status['full_text']
    
    slices = []
    #Strip out the urls.
    if 'urls' in status['entities']:
        for url in status['entities']['urls']:
            slices += [{'start': url['indices'][0], 'stop': url['indices'][1]}]
    
    #Strip out the hashtags.
    if 'hashtags' in status['entities']:
        for tag in status['entities']['hashtags']:
            slices += [{'start': tag['indices'][0], 'stop': tag['indices'][1]}]
    
    #Strip out the user mentions.
    if 'user_mentions' in status['entities']:
        for men in status['entities']['user_mentions']:
            slices += [{'start': men['indices'][0], 'stop': men['indices'][1]}]
    
    #Strip out the media.
    if 'media' in status['entities']:
        for med in status['entities']['media']:
            slices += [{'start': med['indices'][0], 'stop': med['indices'][1]}]
    
    #Strip out the symbols.
    if 'symbols' in status['entities']:
        for sym in status['entities']['symbols']:
            slices += [{'start': sym['indices'][0], 'stop': sym['indices'][1]}]
    
    # Sort the slices from highest start to lowest.
    slices = sorted(slices, key=lambda x: -x['start'])
    
    #No offsets, since we're sorted from highest to lowest.
    for s in slices:
        text = text[:s['start']] + text[s['stop']:]
        
    return text


#method full_clean_tweets()
def full_clean_tweets(tweet):
    stop_words = set(stopwords.words('english'))
    stop_words.update(stopwords.words('spanish'))
    word_tokens = word_tokenize(tweet)

    #after tweepy preprocessing the colon left remain after removing mentions
    #or RT sign in the beginning of the tweet
    tweet = re.sub(r':', '', tweet)
    tweet = re.sub(r'‚Ä¶', '', tweet)
    #replace consecutive non-ASCII characters with a space
    tweet = re.sub(r'[^\x00-\x7F]+',' ', tweet)

    #remove emojis from tweet
    tweet = emoji_pattern.sub(r'', tweet)

    #filter using NLTK library append it to a string
    filtered_tweet = [w for w in word_tokens if not w in stop_words]
    filtered_tweet = []

    #looping through conditions
    for w in word_tokens:
        #check tokens against stop words , emoticons and punctuations
        if w not in stop_words and w not in emoticons and w not in string.punctuation:
            filtered_tweet.append(w)
    return ' '.join(filtered_tweet)


def get_text_sanitized(status):    
    #or RT sign in the beginning of the tweet
    
    tweet = get_text_cleaned(status)
    
    tweet = re.sub(r'RT:', '', tweet)
    tweet = re.sub(r'@', '', tweet)
    
    #replace consecutive non-ASCII characters with a space
    ##note that emojis and emoticons are lost
    #tweet = re.sub(r'[^\x00-\x7F]+',' ', tweet)

    #normalize case
    tweet = tweet.lower()
    
    return tweet


def extract_emojis(str):
    return ''.join(c for c in str if c in emoji.UNICODE_EMOJI)


def strip_non_ascii(string):
    ''' Returns the string without non ASCII characters'''
    stripped = (c for c in string if 0 < ord(c) < 127)
    return ''.join(stripped)




