import re
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import string
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer, SnowballStemmer
from nltk import pos_tag, ne_chunk
from nltk.chunk import tree2conlltags
import os
import seaborn as sns
import matplotlib.pyplot as plt
from collections import Counter

import nltk
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
nltk.download('maxent_ne_chunker')
nltk.download('words')

import warnings
warnings.filterwarnings("ignore")

pd.set_option('display.width', 1000)
pd.options.display.max_colwidth=120
print(os.getcwd())

columns = ["Id","Entity","Target","Text"]
data = pd.read_csv("/app/data/twitter_training.csv", names=columns,header=None)

data.head()

df_train = data[["Text","Target"]]
df_train = df_train.loc[(df_train["Target"]=='Positive') | (df_train["Target"]=='Negative')]
df_train.head()

df_train.info()

df_train= df_train.drop_duplicates()

df_train.info()

sns.countplot(x="Target",data=df_train)

data_val = pd.read_csv("/app/data/twitter_validation.csv", names=columns,header=None)
data_val.head()

df_val = data_val[['Text', 'Target']]
df_val = df_val.loc[(df_val['Target'] == 'Positive') | (df_val['Target'] == 'Negative')]
df_val.head()

df_val.info()

sns.countplot(x="Target",data=df_val)

text_cleaning_re = "@\S+|https?:\S+|http?:\S|[^A-Za-z0-9]+"
emoji_pattern = re.compile("["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           "]+", flags=re.UNICODE)
stemmer = SnowballStemmer('english')
def preprocess(text):
    text = re.sub(text_cleaning_re, ' ', str(text).lower()).strip()
    text = emoji_pattern.sub(r'', text)
    tokens = []
    for token in text.split():
            tokens.append(token)
    return " ".join(tokens)

df_train["Text"] = df_train["Text"].apply(preprocess)
df_train["Text"]= df_train["Text"].str.replace("im","i am")

df_train["Text"].head()

df_val["Text"] = df_val["Text"].apply(preprocess)
df_val["Text"]=df_val["Text"].str.replace("im","i am")
df_val["Text"].head()

