import re
import pandas as pd
import numpy as np
import pickle

from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense, Dropout, Embedding, LSTM
from keras.callbacks import ReduceLROnPlateau, EarlyStopping

from string import punctuation

import gensim

# DATASET
TRAINING_DATA_FILE_PATH = "data/training.1600000.processed.noemoticon.csv"
DATASET_COLUMNS = ["sentiment", "ids", "date", "flag", "user", "text"]
DATASET_ENCODING = "ISO-8859-1"
TRAIN_SIZE = 0.8

# SENTIMENT
POSITIVE = "POSITIVE"
NEGATIVE = "NEGATIVE"
NEUTRAL = "NEUTRAL"
SENTIMENT_THRESHOLDS = (0.4, 0.7)
decode_map = {0: NEGATIVE, 2: NEUTRAL, 4: POSITIVE}

# WORD2VEC
W2V_SIZE = 300
W2V_WINDOW = 7
W2V_EPOCH = 32
W2V_MIN_COUNT = 10

# KERAS
SEQUENCE_LENGTH = 300
EPOCHS = 8
BATCH_SIZE = 1024

# EXPORT
KERAS_MODEL = "data/model.h5"
WORD2VEC_MODEL = "data/model.w2v"
TOKENIZER_MODEL = "data/tokenizer.pkl"
ENCODER_MODEL = "data/encoder.pkl"

dataset = pd.read_csv(TRAINING_DATA_FILE_PATH, encoding=DATASET_ENCODING, names=DATASET_COLUMNS)


def decode_sentiment(label):
    return decode_map[int(label)]


dataset = dataset[['sentiment', 'text']]
dataset['sentiment'] = dataset['sentiment'].apply(lambda x: decode_sentiment(x))

emojis = {':)': 'smile', ':-)': 'smile', ';d': 'wink', ':-E': 'vampire', ':(': 'sad',
          ':-(': 'sad', ':-<': 'sad', ':P': 'raspberry', ':O': 'surprised',
          ':-@': 'shocked', ':@': 'shocked', ':-$': 'confused', ':\\': 'annoyed',
          ':#': 'mute', ':X': 'mute', ':^)': 'smile', ':-&': 'confused', '$_$': 'greedy',
          '@@': 'eyeroll', ':-!': 'confused', ':-D': 'smile', ':-0': 'yell', 'O.o': 'confused',
          '<(-_-)>': 'robot', 'd[-_-]b': 'dj', ":'-)": 'sadsmile', ';)': 'wink',
          ';-)': 'wink', 'O:-)': 'angel', 'O*-)': 'angel', '(:-D': 'gossip', '=^.^=': 'cat'}

stopwords = set(stopwords.words("english") + list(punctuation) + ["USER", "URL"])
wordlemm = WordNetLemmatizer()


def preprocess(tweet):
    # Defining regex patterns.
    url_pa = r"((http://)[^ ]*|(https://)[^ ]*|( www\.)[^ ]*)"
    user_pattern = '@[^\s]+'
    alpha_pattern = "[^a-zA-Z0-9]"
    sequence_pattern = r"(.)\1\1+"
    seq_replace_pattern = r"\1\1"

    tweet = tweet.lower()

    # Replace all URls with 'URL'
    tweet = re.sub(url_pa, ' URL', tweet)
    # Replace all emojis.
    for emoji in emojis.keys():
        tweet = tweet.replace(emoji, "EMOJI" + emojis[emoji])
    # Replace @USERNAME to 'USER'.
    tweet = re.sub(user_pattern, ' USER', tweet)
    # Replace all non alphabets.
    tweet = re.sub(alpha_pattern, " ", tweet)
    # Replace 3 or more consecutive letters by 2 letter.
    tweet = re.sub(sequence_pattern, seq_replace_pattern, tweet)

    tweetwords = ''
    for word in tweet.split():
        # Checking if the word is a stopword.
        if word not in stopwords:
            word = wordlemm.lemmatize(word)
            tweetwords += (word + ' ')
    return tweetwords


dataset['text'] = dataset['text'].apply(lambda x: preprocess(x))

df_train, df_test = train_test_split(dataset, test_size=1 - TRAIN_SIZE, random_state=42)
print("TRAIN size:", len(df_train))
print("TEST size:", len(df_test))

documents = [_text.split() for _text in df_train['text']]
w2v_model = gensim.models.word2vec.Word2Vec(size=W2V_SIZE,
                                            window=W2V_WINDOW,
                                            min_count=W2V_MIN_COUNT,
                                            workers=8)
w2v_model.build_vocab(documents)

words = w2v_model.wv.vocab.keys()
vocab_size = len(words)
print("Vocab size", vocab_size)

w2v_model.train(documents, total_examples=len(documents), epochs=W2V_EPOCH)

tokenizer = Tokenizer()
tokenizer.fit_on_texts(df_train['text'])
vocab_size = len(tokenizer.word_index) + 1
print("Total words", vocab_size)

x_train = pad_sequences(tokenizer.texts_to_sequences(df_train['text']), maxlen=SEQUENCE_LENGTH)
x_test = pad_sequences(tokenizer.texts_to_sequences(df_test['text']), maxlen=SEQUENCE_LENGTH)

labels = df_train['sentiment'].unique().tolist()
labels.append(NEUTRAL)

encoder = LabelEncoder()
encoder.fit(df_train['sentiment'].tolist())

y_train = encoder.transform(df_train['sentiment'].tolist())
y_test = encoder.transform(df_test['sentiment'].tolist())

y_train = y_train.reshape(-1, 1)
y_test = y_test.reshape(-1, 1)

print("y_train", y_train.shape)
print("y_test", y_test.shape)

print("x_train", x_train.shape)
print("y_train", y_train.shape)
print()
print("x_test", x_test.shape)
print("y_test", y_test.shape)

embedding_matrix = np.zeros((vocab_size, W2V_SIZE))
for word, i in tokenizer.word_index.items():
    if word in w2v_model.wv:
        embedding_matrix[i] = w2v_model.wv[word]
print(embedding_matrix.shape)

embedding_layer = Embedding(vocab_size, W2V_SIZE, weights=[embedding_matrix], input_length=SEQUENCE_LENGTH,
                            trainable=False)

# Build model
model = Sequential()
model.add(embedding_layer)
model.add(Dropout(0.5))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))

model.summary()

model.compile(loss='binary_crossentropy',
              optimizer="adam",
              metrics=['accuracy'])

callbacks = [ReduceLROnPlateau(monitor='val_loss', patience=5, cooldown=0),
             EarlyStopping(monitor='val_accuracy', min_delta=1e-4, patience=5)]

model.fit(x_train, y_train,
          batch_size=BATCH_SIZE,
          epochs=EPOCHS,
          validation_split=0.1,
          verbose=1,
          callbacks=callbacks)

score = model.evaluate(x_test, y_test, batch_size=BATCH_SIZE)
print()
print("ACCURACY:", score[1])
print("LOSS:", score[0])

model.save(KERAS_MODEL)
w2v_model.save(WORD2VEC_MODEL)
pickle.dump(tokenizer, open(TOKENIZER_MODEL, "wb"), protocol=0)
pickle.dump(encoder, open(ENCODER_MODEL, "wb"), protocol=0)

32493ef4ac08386abb64cbd1db7532d92f003074
<mask token>


def decode_sentiment(label):
    return decode_map[int(label)]


<mask token>


def preprocess(tweet):
    url_pa = '((http://)[^ ]*|(https://)[^ ]*|( www\\.)[^ ]*)'
    user_pattern = '@[^\\s]+'
    alpha_pattern = '[^a-zA-Z0-9]'
    sequence_pattern = '(.)\\1\\1+'
    seq_replace_pattern = '\\1\\1'
    tweet = tweet.lower()
    tweet = re.sub(url_pa, ' URL', tweet)
    for emoji in emojis.keys():
        tweet = tweet.replace(emoji, 'EMOJI' + emojis[emoji])
    tweet = re.sub(user_pattern, ' USER', tweet)
    tweet = re.sub(alpha_pattern, ' ', tweet)
    tweet = re.sub(sequence_pattern, seq_replace_pattern, tweet)
    tweetwords = ''
    for word in tweet.split():
        if word not in stopwords:
            word = wordlemm.lemmatize(word)
            tweetwords += word + ' '
    return tweetwords


<mask token>

<mask token>


def decode_sentiment(label):
    return decode_map[int(label)]


<mask token>


def preprocess(tweet):
    url_pa = '((http://)[^ ]*|(https://)[^ ]*|( www\\.)[^ ]*)'
    user_pattern = '@[^\\s]+'
    alpha_pattern = '[^a-zA-Z0-9]'
    sequence_pattern = '(.)\\1\\1+'
    seq_replace_pattern = '\\1\\1'
    tweet = tweet.lower()
    tweet = re.sub(url_pa, ' URL', tweet)
    for emoji in emojis.keys():
        tweet = tweet.replace(emoji, 'EMOJI' + emojis[emoji])
    tweet = re.sub(user_pattern, ' USER', tweet)
    tweet = re.sub(alpha_pattern, ' ', tweet)
    tweet = re.sub(sequence_pattern, seq_replace_pattern, tweet)
    tweetwords = ''
    for word in tweet.split():
        if word not in stopwords:
            word = wordlemm.lemmatize(word)
            tweetwords += word + ' '
    return tweetwords


<mask token>
print('TRAIN size:', len(df_train))
print('TEST size:', len(df_test))
<mask token>
w2v_model.build_vocab(documents)
<mask token>
print('Vocab size', vocab_size)
w2v_model.train(documents, total_examples=len(documents), epochs=W2V_EPOCH)
<mask token>
tokenizer.fit_on_texts(df_train['text'])
<mask token>
print('Total words', vocab_size)
<mask token>
labels.append(NEUTRAL)
<mask token>
encoder.fit(df_train['sentiment'].tolist())
<mask token>
print('y_train', y_train.shape)
print('y_test', y_test.shape)
print('x_train', x_train.shape)
print('y_train', y_train.shape)
print()
print('x_test', x_test.shape)
print('y_test', y_test.shape)
<mask token>
for word, i in tokenizer.word_index.items():
    if word in w2v_model.wv:
        embedding_matrix[i] = w2v_model.wv[word]
print(embedding_matrix.shape)
<mask token>
model.add(embedding_layer)
model.add(Dropout(0.5))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[
    'accuracy'])
<mask token>
model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS,
    validation_split=0.1, verbose=1, callbacks=callbacks)
<mask token>
print()
print('ACCURACY:', score[1])
print('LOSS:', score[0])
model.save(KERAS_MODEL)
w2v_model.save(WORD2VEC_MODEL)
pickle.dump(tokenizer, open(TOKENIZER_MODEL, 'wb'), protocol=0)
pickle.dump(encoder, open(ENCODER_MODEL, 'wb'), protocol=0)

<mask token>
TRAINING_DATA_FILE_PATH = 'data/training.1600000.processed.noemoticon.csv'
DATASET_COLUMNS = ['sentiment', 'ids', 'date', 'flag', 'user', 'text']
DATASET_ENCODING = 'ISO-8859-1'
TRAIN_SIZE = 0.8
POSITIVE = 'POSITIVE'
NEGATIVE = 'NEGATIVE'
NEUTRAL = 'NEUTRAL'
SENTIMENT_THRESHOLDS = 0.4, 0.7
decode_map = {(0): NEGATIVE, (2): NEUTRAL, (4): POSITIVE}
W2V_SIZE = 300
W2V_WINDOW = 7
W2V_EPOCH = 32
W2V_MIN_COUNT = 10
SEQUENCE_LENGTH = 300
EPOCHS = 8
BATCH_SIZE = 1024
KERAS_MODEL = 'data/model.h5'
WORD2VEC_MODEL = 'data/model.w2v'
TOKENIZER_MODEL = 'data/tokenizer.pkl'
ENCODER_MODEL = 'data/encoder.pkl'
dataset = pd.read_csv(TRAINING_DATA_FILE_PATH, encoding=DATASET_ENCODING,
    names=DATASET_COLUMNS)


def decode_sentiment(label):
    return decode_map[int(label)]


dataset = dataset[['sentiment', 'text']]
dataset['sentiment'] = dataset['sentiment'].apply(lambda x: decode_sentiment(x)
    )
emojis = {':)': 'smile', ':-)': 'smile', ';d': 'wink', ':-E': 'vampire',
    ':(': 'sad', ':-(': 'sad', ':-<': 'sad', ':P': 'raspberry', ':O':
    'surprised', ':-@': 'shocked', ':@': 'shocked', ':-$': 'confused',
    ':\\': 'annoyed', ':#': 'mute', ':X': 'mute', ':^)': 'smile', ':-&':
    'confused', '$_$': 'greedy', '@@': 'eyeroll', ':-!': 'confused', ':-D':
    'smile', ':-0': 'yell', 'O.o': 'confused', '<(-_-)>': 'robot',
    'd[-_-]b': 'dj', ":'-)": 'sadsmile', ';)': 'wink', ';-)': 'wink',
    'O:-)': 'angel', 'O*-)': 'angel', '(:-D': 'gossip', '=^.^=': 'cat'}
stopwords = set(stopwords.words('english') + list(punctuation) + ['USER',
    'URL'])
wordlemm = WordNetLemmatizer()


def preprocess(tweet):
    url_pa = '((http://)[^ ]*|(https://)[^ ]*|( www\\.)[^ ]*)'
    user_pattern = '@[^\\s]+'
    alpha_pattern = '[^a-zA-Z0-9]'
    sequence_pattern = '(.)\\1\\1+'
    seq_replace_pattern = '\\1\\1'
    tweet = tweet.lower()
    tweet = re.sub(url_pa, ' URL', tweet)
    for emoji in emojis.keys():
        tweet = tweet.replace(emoji, 'EMOJI' + emojis[emoji])
    tweet = re.sub(user_pattern, ' USER', tweet)
    tweet = re.sub(alpha_pattern, ' ', tweet)
    tweet = re.sub(sequence_pattern, seq_replace_pattern, tweet)
    tweetwords = ''
    for word in tweet.split():
        if word not in stopwords:
            word = wordlemm.lemmatize(word)
            tweetwords += word + ' '
    return tweetwords


dataset['text'] = dataset['text'].apply(lambda x: preprocess(x))
df_train, df_test = train_test_split(dataset, test_size=1 - TRAIN_SIZE,
    random_state=42)
print('TRAIN size:', len(df_train))
print('TEST size:', len(df_test))
documents = [_text.split() for _text in df_train['text']]
w2v_model = gensim.models.word2vec.Word2Vec(size=W2V_SIZE, window=
    W2V_WINDOW, min_count=W2V_MIN_COUNT, workers=8)
w2v_model.build_vocab(documents)
words = w2v_model.wv.vocab.keys()
vocab_size = len(words)
print('Vocab size', vocab_size)
w2v_model.train(documents, total_examples=len(documents), epochs=W2V_EPOCH)
tokenizer = Tokenizer()
tokenizer.fit_on_texts(df_train['text'])
vocab_size = len(tokenizer.word_index) + 1
print('Total words', vocab_size)
x_train = pad_sequences(tokenizer.texts_to_sequences(df_train['text']),
    maxlen=SEQUENCE_LENGTH)
x_test = pad_sequences(tokenizer.texts_to_sequences(df_test['text']),
    maxlen=SEQUENCE_LENGTH)
labels = df_train['sentiment'].unique().tolist()
labels.append(NEUTRAL)
encoder = LabelEncoder()
encoder.fit(df_train['sentiment'].tolist())
y_train = encoder.transform(df_train['sentiment'].tolist())
y_test = encoder.transform(df_test['sentiment'].tolist())
y_train = y_train.reshape(-1, 1)
y_test = y_test.reshape(-1, 1)
print('y_train', y_train.shape)
print('y_test', y_test.shape)
print('x_train', x_train.shape)
print('y_train', y_train.shape)
print()
print('x_test', x_test.shape)
print('y_test', y_test.shape)
embedding_matrix = np.zeros((vocab_size, W2V_SIZE))
for word, i in tokenizer.word_index.items():
    if word in w2v_model.wv:
        embedding_matrix[i] = w2v_model.wv[word]
print(embedding_matrix.shape)
embedding_layer = Embedding(vocab_size, W2V_SIZE, weights=[embedding_matrix
    ], input_length=SEQUENCE_LENGTH, trainable=False)
model = Sequential()
model.add(embedding_layer)
model.add(Dropout(0.5))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[
    'accuracy'])
callbacks = [ReduceLROnPlateau(monitor='val_loss', patience=5, cooldown=0),
    EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=5)]
model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS,
    validation_split=0.1, verbose=1, callbacks=callbacks)
score = model.evaluate(x_test, y_test, batch_size=BATCH_SIZE)
print()
print('ACCURACY:', score[1])
print('LOSS:', score[0])
model.save(KERAS_MODEL)
w2v_model.save(WORD2VEC_MODEL)
pickle.dump(tokenizer, open(TOKENIZER_MODEL, 'wb'), protocol=0)
pickle.dump(encoder, open(ENCODER_MODEL, 'wb'), protocol=0)

import re
import pandas as pd
import numpy as np
import pickle
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense, Dropout, Embedding, LSTM
from keras.callbacks import ReduceLROnPlateau, EarlyStopping
from string import punctuation
import gensim
TRAINING_DATA_FILE_PATH = 'data/training.1600000.processed.noemoticon.csv'
DATASET_COLUMNS = ['sentiment', 'ids', 'date', 'flag', 'user', 'text']
DATASET_ENCODING = 'ISO-8859-1'
TRAIN_SIZE = 0.8
POSITIVE = 'POSITIVE'
NEGATIVE = 'NEGATIVE'
NEUTRAL = 'NEUTRAL'
SENTIMENT_THRESHOLDS = 0.4, 0.7
decode_map = {(0): NEGATIVE, (2): NEUTRAL, (4): POSITIVE}
W2V_SIZE = 300
W2V_WINDOW = 7
W2V_EPOCH = 32
W2V_MIN_COUNT = 10
SEQUENCE_LENGTH = 300
EPOCHS = 8
BATCH_SIZE = 1024
KERAS_MODEL = 'data/model.h5'
WORD2VEC_MODEL = 'data/model.w2v'
TOKENIZER_MODEL = 'data/tokenizer.pkl'
ENCODER_MODEL = 'data/encoder.pkl'
dataset = pd.read_csv(TRAINING_DATA_FILE_PATH, encoding=DATASET_ENCODING,
    names=DATASET_COLUMNS)


def decode_sentiment(label):
    return decode_map[int(label)]


dataset = dataset[['sentiment', 'text']]
dataset['sentiment'] = dataset['sentiment'].apply(lambda x: decode_sentiment(x)
    )
emojis = {':)': 'smile', ':-)': 'smile', ';d': 'wink', ':-E': 'vampire',
    ':(': 'sad', ':-(': 'sad', ':-<': 'sad', ':P': 'raspberry', ':O':
    'surprised', ':-@': 'shocked', ':@': 'shocked', ':-$': 'confused',
    ':\\': 'annoyed', ':#': 'mute', ':X': 'mute', ':^)': 'smile', ':-&':
    'confused', '$_$': 'greedy', '@@': 'eyeroll', ':-!': 'confused', ':-D':
    'smile', ':-0': 'yell', 'O.o': 'confused', '<(-_-)>': 'robot',
    'd[-_-]b': 'dj', ":'-)": 'sadsmile', ';)': 'wink', ';-)': 'wink',
    'O:-)': 'angel', 'O*-)': 'angel', '(:-D': 'gossip', '=^.^=': 'cat'}
stopwords = set(stopwords.words('english') + list(punctuation) + ['USER',
    'URL'])
wordlemm = WordNetLemmatizer()


def preprocess(tweet):
    url_pa = '((http://)[^ ]*|(https://)[^ ]*|( www\\.)[^ ]*)'
    user_pattern = '@[^\\s]+'
    alpha_pattern = '[^a-zA-Z0-9]'
    sequence_pattern = '(.)\\1\\1+'
    seq_replace_pattern = '\\1\\1'
    tweet = tweet.lower()
    tweet = re.sub(url_pa, ' URL', tweet)
    for emoji in emojis.keys():
        tweet = tweet.replace(emoji, 'EMOJI' + emojis[emoji])
    tweet = re.sub(user_pattern, ' USER', tweet)
    tweet = re.sub(alpha_pattern, ' ', tweet)
    tweet = re.sub(sequence_pattern, seq_replace_pattern, tweet)
    tweetwords = ''
    for word in tweet.split():
        if word not in stopwords:
            word = wordlemm.lemmatize(word)
            tweetwords += word + ' '
    return tweetwords


dataset['text'] = dataset['text'].apply(lambda x: preprocess(x))
df_train, df_test = train_test_split(dataset, test_size=1 - TRAIN_SIZE,
    random_state=42)
print('TRAIN size:', len(df_train))
print('TEST size:', len(df_test))
documents = [_text.split() for _text in df_train['text']]
w2v_model = gensim.models.word2vec.Word2Vec(size=W2V_SIZE, window=
    W2V_WINDOW, min_count=W2V_MIN_COUNT, workers=8)
w2v_model.build_vocab(documents)
words = w2v_model.wv.vocab.keys()
vocab_size = len(words)
print('Vocab size', vocab_size)
w2v_model.train(documents, total_examples=len(documents), epochs=W2V_EPOCH)
tokenizer = Tokenizer()
tokenizer.fit_on_texts(df_train['text'])
vocab_size = len(tokenizer.word_index) + 1
print('Total words', vocab_size)
x_train = pad_sequences(tokenizer.texts_to_sequences(df_train['text']),
    maxlen=SEQUENCE_LENGTH)
x_test = pad_sequences(tokenizer.texts_to_sequences(df_test['text']),
    maxlen=SEQUENCE_LENGTH)
labels = df_train['sentiment'].unique().tolist()
labels.append(NEUTRAL)
encoder = LabelEncoder()
encoder.fit(df_train['sentiment'].tolist())
y_train = encoder.transform(df_train['sentiment'].tolist())
y_test = encoder.transform(df_test['sentiment'].tolist())
y_train = y_train.reshape(-1, 1)
y_test = y_test.reshape(-1, 1)
print('y_train', y_train.shape)
print('y_test', y_test.shape)
print('x_train', x_train.shape)
print('y_train', y_train.shape)
print()
print('x_test', x_test.shape)
print('y_test', y_test.shape)
embedding_matrix = np.zeros((vocab_size, W2V_SIZE))
for word, i in tokenizer.word_index.items():
    if word in w2v_model.wv:
        embedding_matrix[i] = w2v_model.wv[word]
print(embedding_matrix.shape)
embedding_layer = Embedding(vocab_size, W2V_SIZE, weights=[embedding_matrix
    ], input_length=SEQUENCE_LENGTH, trainable=False)
model = Sequential()
model.add(embedding_layer)
model.add(Dropout(0.5))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[
    'accuracy'])
callbacks = [ReduceLROnPlateau(monitor='val_loss', patience=5, cooldown=0),
    EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=5)]
model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS,
    validation_split=0.1, verbose=1, callbacks=callbacks)
score = model.evaluate(x_test, y_test, batch_size=BATCH_SIZE)
print()
print('ACCURACY:', score[1])
print('LOSS:', score[0])
model.save(KERAS_MODEL)
w2v_model.save(WORD2VEC_MODEL)
pickle.dump(tokenizer, open(TOKENIZER_MODEL, 'wb'), protocol=0)
pickle.dump(encoder, open(ENCODER_MODEL, 'wb'), protocol=0)

import re
import pandas as pd
import numpy as np
import pickle

from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense, Dropout, Embedding, LSTM
from keras.callbacks import ReduceLROnPlateau, EarlyStopping

from string import punctuation

import gensim

# DATASET
TRAINING_DATA_FILE_PATH = "data/training.1600000.processed.noemoticon.csv"
DATASET_COLUMNS = ["sentiment", "ids", "date", "flag", "user", "text"]
DATASET_ENCODING = "ISO-8859-1"
TRAIN_SIZE = 0.8

# SENTIMENT
POSITIVE = "POSITIVE"
NEGATIVE = "NEGATIVE"
NEUTRAL = "NEUTRAL"
SENTIMENT_THRESHOLDS = (0.4, 0.7)
decode_map = {0: NEGATIVE, 2: NEUTRAL, 4: POSITIVE}

# WORD2VEC
W2V_SIZE = 300
W2V_WINDOW = 7
W2V_EPOCH = 32
W2V_MIN_COUNT = 10

# KERAS
SEQUENCE_LENGTH = 300
EPOCHS = 8
BATCH_SIZE = 1024

# EXPORT
KERAS_MODEL = "data/model.h5"
WORD2VEC_MODEL = "data/model.w2v"
TOKENIZER_MODEL = "data/tokenizer.pkl"
ENCODER_MODEL = "data/encoder.pkl"

dataset = pd.read_csv(TRAINING_DATA_FILE_PATH, encoding=DATASET_ENCODING, names=DATASET_COLUMNS)


def decode_sentiment(label):
    return decode_map[int(label)]


dataset = dataset[['sentiment', 'text']]
dataset['sentiment'] = dataset['sentiment'].apply(lambda x: decode_sentiment(x))

emojis = {':)': 'smile', ':-)': 'smile', ';d': 'wink', ':-E': 'vampire', ':(': 'sad',
          ':-(': 'sad', ':-<': 'sad', ':P': 'raspberry', ':O': 'surprised',
          ':-@': 'shocked', ':@': 'shocked', ':-$': 'confused', ':\\': 'annoyed',
          ':#': 'mute', ':X': 'mute', ':^)': 'smile', ':-&': 'confused', '$_$': 'greedy',
          '@@': 'eyeroll', ':-!': 'confused', ':-D': 'smile', ':-0': 'yell', 'O.o': 'confused',
          '<(-_-)>': 'robot', 'd[-_-]b': 'dj', ":'-)": 'sadsmile', ';)': 'wink',
          ';-)': 'wink', 'O:-)': 'angel', 'O*-)': 'angel', '(:-D': 'gossip', '=^.^=': 'cat'}

stopwords = set(stopwords.words("english") + list(punctuation) + ["USER", "URL"])
wordlemm = WordNetLemmatizer()


def preprocess(tweet):
    # Defining regex patterns.
    url_pa = r"((http://)[^ ]*|(https://)[^ ]*|( www\.)[^ ]*)"
    user_pattern = '@[^\s]+'
    alpha_pattern = "[^a-zA-Z0-9]"
    sequence_pattern = r"(.)\1\1+"
    seq_replace_pattern = r"\1\1"

    tweet = tweet.lower()

    # Replace all URls with 'URL'
    tweet = re.sub(url_pa, ' URL', tweet)
    # Replace all emojis.
    for emoji in emojis.keys():
        tweet = tweet.replace(emoji, "EMOJI" + emojis[emoji])
    # Replace @USERNAME to 'USER'.
    tweet = re.sub(user_pattern, ' USER', tweet)
    # Replace all non alphabets.
    tweet = re.sub(alpha_pattern, " ", tweet)
    # Replace 3 or more consecutive letters by 2 letter.
    tweet = re.sub(sequence_pattern, seq_replace_pattern, tweet)

    tweetwords = ''
    for word in tweet.split():
        # Checking if the word is a stopword.
        if word not in stopwords:
            word = wordlemm.lemmatize(word)
            tweetwords += (word + ' ')
    return tweetwords


dataset['text'] = dataset['text'].apply(lambda x: preprocess(x))

df_train, df_test = train_test_split(dataset, test_size=1 - TRAIN_SIZE, random_state=42)
print("TRAIN size:", len(df_train))
print("TEST size:", len(df_test))

documents = [_text.split() for _text in df_train['text']]
w2v_model = gensim.models.word2vec.Word2Vec(size=W2V_SIZE,
                                            window=W2V_WINDOW,
                                            min_count=W2V_MIN_COUNT,
                                            workers=8)
w2v_model.build_vocab(documents)

words = w2v_model.wv.vocab.keys()
vocab_size = len(words)
print("Vocab size", vocab_size)

w2v_model.train(documents, total_examples=len(documents), epochs=W2V_EPOCH)

tokenizer = Tokenizer()
tokenizer.fit_on_texts(df_train['text'])
vocab_size = len(tokenizer.word_index) + 1
print("Total words", vocab_size)

x_train = pad_sequences(tokenizer.texts_to_sequences(df_train['text']), maxlen=SEQUENCE_LENGTH)
x_test = pad_sequences(tokenizer.texts_to_sequences(df_test['text']), maxlen=SEQUENCE_LENGTH)

labels = df_train['sentiment'].unique().tolist()
labels.append(NEUTRAL)

encoder = LabelEncoder()
encoder.fit(df_train['sentiment'].tolist())

y_train = encoder.transform(df_train['sentiment'].tolist())
y_test = encoder.transform(df_test['sentiment'].tolist())

y_train = y_train.reshape(-1, 1)
y_test = y_test.reshape(-1, 1)

print("y_train", y_train.shape)
print("y_test", y_test.shape)

print("x_train", x_train.shape)
print("y_train", y_train.shape)
print()
print("x_test", x_test.shape)
print("y_test", y_test.shape)

embedding_matrix = np.zeros((vocab_size, W2V_SIZE))
for word, i in tokenizer.word_index.items():
    if word in w2v_model.wv:
        embedding_matrix[i] = w2v_model.wv[word]
print(embedding_matrix.shape)

embedding_layer = Embedding(vocab_size, W2V_SIZE, weights=[embedding_matrix], input_length=SEQUENCE_LENGTH,
                            trainable=False)

# Build model
model = Sequential()
model.add(embedding_layer)
model.add(Dropout(0.5))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))

model.summary()

model.compile(loss='binary_crossentropy',
              optimizer="adam",
              metrics=['accuracy'])

callbacks = [ReduceLROnPlateau(monitor='val_loss', patience=5, cooldown=0),
             EarlyStopping(monitor='val_accuracy', min_delta=1e-4, patience=5)]

model.fit(x_train, y_train,
          batch_size=BATCH_SIZE,
          epochs=EPOCHS,
          validation_split=0.1,
          verbose=1,
          callbacks=callbacks)

score = model.evaluate(x_test, y_test, batch_size=BATCH_SIZE)
print()
print("ACCURACY:", score[1])
print("LOSS:", score[0])

model.save(KERAS_MODEL)
w2v_model.save(WORD2VEC_MODEL)
pickle.dump(tokenizer, open(TOKENIZER_MODEL, "wb"), protocol=0)
pickle.dump(encoder, open(ENCODER_MODEL, "wb"), protocol=0)
