# -*- coding: utf-8 -*-
"""Dicoding_Submission_Intermediate_NLP#1.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1wuIaJJQ11A_C4yVEBE7aCX0BGI86nRJA
"""

from google.colab import drive
drive.mount('/content/drive')

import pandas as pd
df = pd.read_csv('/content/drive/My Drive/Womens Clothing E-Commerce Reviews.csv')

df.head()

"""dataset https://www.kaggle.com/nicapotato/womens-ecommerce-clothing-reviews"""

len(df['Class Name'].unique())

df['Class Name'].unique()

df['Department Name'].value_counts()

df1 = df.drop(columns=['Unnamed: 0', 'Clothing ID', 'Age','Title', 'Rating', 'Recommended IND', 'Positive Feedback Count', 'Division Name',  'Department Name'])

df2 = df1.loc[(df['Class Name'] == 'Pants') | (df['Class Name'] == 'Jeans') | (df['Class Name'] == 'Fine gauge') ]

df2.head()

df2.columns = df2.columns.str.lower()
df2['review text'] = df2['review text'].astype(str)

# fungsi untuk text cleaning
import re

def clean_text(text):
    text = text.lower()
    text = re.sub(r"what's", "what is ", text)
    text = re.sub(r"\'s", " ", text)
    text = re.sub(r"\'ve", " have ", text)
    text = re.sub(r"can't", "can not ", text)
    text = re.sub(r"n't", " not ", text)
    text = re.sub(r"i'm", "i am ", text)
    text = re.sub(r"\'re", " are ", text)
    text = re.sub(r"\'d", " would ", text)
    text = re.sub(r"\'ll", " will ", text)
    text = re.sub(r"\'scuse", " excuse ", text)
    #text = re.sub('\W', ' ', text)
    #text = re.sub('\s+', ' ', text)
    text = text.strip(' ')
    return text

df2['review text'] = df2['review text'].apply(clean_text)

# Commented out IPython magic to ensure Python compatibility.
import matplotlib.pyplot as plt
# %matplotlib inline
import seaborn as sns

sns.countplot(df2['class name'])
plt.xlabel('class name')

"""Data cukup balanced sehingga cocok digunakan sebagai klasifikasi"""

outfit = pd.get_dummies(df2['class name'])
fit = pd.concat([df2, outfit], axis=1)
fit = fit.drop(columns=['class name'])

fit

review = fit['review text'].values
label = fit[['Fine gauge', 'Jeans', 'Pants']].values

review

label

from sklearn.model_selection import train_test_split
review_train, review_test, label_train, label_test = train_test_split(review, label, test_size=0.2, shuffle=True)

import numpy as np
from gensim.parsing.porter import PorterStemmer
from gensim.parsing.preprocessing import remove_stopwords

porterStemmer = PorterStemmer()

rev_train_texts = []
for text in review_train:
  removed_text = remove_stopwords(text)   
  removed_text = porterStemmer.stem_sentence(removed_text) 
  rev_train_texts.append(removed_text)

rev_train_texts = np.array(rev_train_texts, dtype='O')  

rev_test_texts = []
for text in review_test:
  removed_text = remove_stopwords(text)
  removed_text = porterStemmer.stem_sentence(removed_text)
  rev_test_texts.append(removed_text)

rev_test_texts = np.array(rev_test_texts, dtype='O')

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
 
tokenizer = Tokenizer(num_words=5000, oov_token='x')
tokenizer.fit_on_texts(rev_train_texts) 
tokenizer.fit_on_texts(rev_test_texts)
	 
seq_train = tokenizer.texts_to_sequences(rev_train_texts)
seq_test = tokenizer.texts_to_sequences(rev_test_texts)
	 
padded_train = pad_sequences(seq_train) 
padded_test = pad_sequences(seq_test)

word_to_index = tokenizer.word_index

len(word_to_index)

import tensorflow as tf

model = tf.keras.Sequential([
    tf.keras.layers.Embedding(input_dim=5000, output_dim=32),
    tf.keras.layers.LSTM(64, dropout=0.1, recurrent_dropout=0.1),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Dense(128,activation = 'relu'),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Dense(64,activation = 'relu'),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Dense(32,activation = 'relu'),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Dense(3, activation='softmax')
])

model.summary()

opt = tf.keras.optimizers.Adam(learning_rate=0.001)
model.compile(loss = 'categorical_crossentropy', optimizer = opt, metrics = ['acc'])

from keras.callbacks import EarlyStopping,ReduceLROnPlateau

reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
                              patience=1, min_lr=0.00001)

early_stopping_monitor = EarlyStopping(monitor = 'val_acc',
                          min_delta = 0,
                          patience = 10,
                          verbose = 1,
                          restore_best_weights = True)

hist = model.fit(padded_train,label_train,steps_per_epoch=25,epochs=50,validation_data=(padded_test, label_test),validation_steps=5,verbose=1, callbacks=[early_stopping_monitor,reduce_lr])

#Visualize the model accuracy
plt.plot(hist.history['acc'])
plt.plot(hist.history['val_acc'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper left')
plt.show()

#Visualize the model loss
plt.plot(hist.history['loss'])
plt.plot(hist.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper right')
plt.show()
7b07d3935b3f47b4dc094bb44f4aec21a0ea46a2
<mask token>

<mask token>
drive.mount('/content/drive')
<mask token>
df.head()
<mask token>
len(df['Class Name'].unique())
df['Class Name'].unique()
df['Department Name'].value_counts()
<mask token>
df2.head()
<mask token>


def clean_text(text):
    text = text.lower()
    text = re.sub("what's", 'what is ', text)
    text = re.sub("\\'s", ' ', text)
    text = re.sub("\\'ve", ' have ', text)
    text = re.sub("can't", 'can not ', text)
    text = re.sub("n't", ' not ', text)
    text = re.sub("i'm", 'i am ', text)
    text = re.sub("\\'re", ' are ', text)
    text = re.sub("\\'d", ' would ', text)
    text = re.sub("\\'ll", ' will ', text)
    text = re.sub("\\'scuse", ' excuse ', text)
    text = text.strip(' ')
    return text


<mask token>
sns.countplot(df2['class name'])
plt.xlabel('class name')
<mask token>
fit
<mask token>
review
label
<mask token>
for text in review_train:
    removed_text = remove_stopwords(text)
    removed_text = porterStemmer.stem_sentence(removed_text)
    rev_train_texts.append(removed_text)
<mask token>
for text in review_test:
    removed_text = remove_stopwords(text)
    removed_text = porterStemmer.stem_sentence(removed_text)
    rev_test_texts.append(removed_text)
<mask token>
tokenizer.fit_on_texts(rev_train_texts)
tokenizer.fit_on_texts(rev_test_texts)
<mask token>
len(word_to_index)
<mask token>
model.summary()
<mask token>
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['acc'])
<mask token>
plt.plot(hist.history['acc'])
plt.plot(hist.history['val_acc'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper left')
plt.show()
plt.plot(hist.history['loss'])
plt.plot(hist.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper right')
plt.show()

<mask token>
drive.mount('/content/drive')
<mask token>
df = pd.read_csv(
    '/content/drive/My Drive/Womens Clothing E-Commerce Reviews.csv')
df.head()
<mask token>
len(df['Class Name'].unique())
df['Class Name'].unique()
df['Department Name'].value_counts()
df1 = df.drop(columns=['Unnamed: 0', 'Clothing ID', 'Age', 'Title',
    'Rating', 'Recommended IND', 'Positive Feedback Count', 'Division Name',
    'Department Name'])
df2 = df1.loc[(df['Class Name'] == 'Pants') | (df['Class Name'] == 'Jeans') |
    (df['Class Name'] == 'Fine gauge')]
df2.head()
df2.columns = df2.columns.str.lower()
df2['review text'] = df2['review text'].astype(str)
<mask token>


def clean_text(text):
    text = text.lower()
    text = re.sub("what's", 'what is ', text)
    text = re.sub("\\'s", ' ', text)
    text = re.sub("\\'ve", ' have ', text)
    text = re.sub("can't", 'can not ', text)
    text = re.sub("n't", ' not ', text)
    text = re.sub("i'm", 'i am ', text)
    text = re.sub("\\'re", ' are ', text)
    text = re.sub("\\'d", ' would ', text)
    text = re.sub("\\'ll", ' will ', text)
    text = re.sub("\\'scuse", ' excuse ', text)
    text = text.strip(' ')
    return text


df2['review text'] = df2['review text'].apply(clean_text)
<mask token>
sns.countplot(df2['class name'])
plt.xlabel('class name')
<mask token>
outfit = pd.get_dummies(df2['class name'])
fit = pd.concat([df2, outfit], axis=1)
fit = fit.drop(columns=['class name'])
fit
review = fit['review text'].values
label = fit[['Fine gauge', 'Jeans', 'Pants']].values
review
label
<mask token>
review_train, review_test, label_train, label_test = train_test_split(review,
    label, test_size=0.2, shuffle=True)
<mask token>
porterStemmer = PorterStemmer()
rev_train_texts = []
for text in review_train:
    removed_text = remove_stopwords(text)
    removed_text = porterStemmer.stem_sentence(removed_text)
    rev_train_texts.append(removed_text)
rev_train_texts = np.array(rev_train_texts, dtype='O')
rev_test_texts = []
for text in review_test:
    removed_text = remove_stopwords(text)
    removed_text = porterStemmer.stem_sentence(removed_text)
    rev_test_texts.append(removed_text)
rev_test_texts = np.array(rev_test_texts, dtype='O')
<mask token>
tokenizer = Tokenizer(num_words=5000, oov_token='x')
tokenizer.fit_on_texts(rev_train_texts)
tokenizer.fit_on_texts(rev_test_texts)
seq_train = tokenizer.texts_to_sequences(rev_train_texts)
seq_test = tokenizer.texts_to_sequences(rev_test_texts)
padded_train = pad_sequences(seq_train)
padded_test = pad_sequences(seq_test)
word_to_index = tokenizer.word_index
len(word_to_index)
<mask token>
model = tf.keras.Sequential([tf.keras.layers.Embedding(input_dim=5000,
    output_dim=32), tf.keras.layers.LSTM(64, dropout=0.1, recurrent_dropout
    =0.1), tf.keras.layers.Flatten(), tf.keras.layers.Dropout(0.2), tf.
    keras.layers.Dense(128, activation='relu'), tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(64, activation=
    'relu'), tf.keras.layers.Flatten(), tf.keras.layers.Dropout(0.2), tf.
    keras.layers.Dense(32, activation='relu'), tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(3, activation=
    'softmax')])
model.summary()
opt = tf.keras.optimizers.Adam(learning_rate=0.001)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['acc'])
<mask token>
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=1,
    min_lr=1e-05)
early_stopping_monitor = EarlyStopping(monitor='val_acc', min_delta=0,
    patience=10, verbose=1, restore_best_weights=True)
hist = model.fit(padded_train, label_train, steps_per_epoch=25, epochs=50,
    validation_data=(padded_test, label_test), validation_steps=5, verbose=
    1, callbacks=[early_stopping_monitor, reduce_lr])
plt.plot(hist.history['acc'])
plt.plot(hist.history['val_acc'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper left')
plt.show()
plt.plot(hist.history['loss'])
plt.plot(hist.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper right')
plt.show()

<mask token>
from google.colab import drive
drive.mount('/content/drive')
import pandas as pd
df = pd.read_csv(
    '/content/drive/My Drive/Womens Clothing E-Commerce Reviews.csv')
df.head()
<mask token>
len(df['Class Name'].unique())
df['Class Name'].unique()
df['Department Name'].value_counts()
df1 = df.drop(columns=['Unnamed: 0', 'Clothing ID', 'Age', 'Title',
    'Rating', 'Recommended IND', 'Positive Feedback Count', 'Division Name',
    'Department Name'])
df2 = df1.loc[(df['Class Name'] == 'Pants') | (df['Class Name'] == 'Jeans') |
    (df['Class Name'] == 'Fine gauge')]
df2.head()
df2.columns = df2.columns.str.lower()
df2['review text'] = df2['review text'].astype(str)
import re


def clean_text(text):
    text = text.lower()
    text = re.sub("what's", 'what is ', text)
    text = re.sub("\\'s", ' ', text)
    text = re.sub("\\'ve", ' have ', text)
    text = re.sub("can't", 'can not ', text)
    text = re.sub("n't", ' not ', text)
    text = re.sub("i'm", 'i am ', text)
    text = re.sub("\\'re", ' are ', text)
    text = re.sub("\\'d", ' would ', text)
    text = re.sub("\\'ll", ' will ', text)
    text = re.sub("\\'scuse", ' excuse ', text)
    text = text.strip(' ')
    return text


df2['review text'] = df2['review text'].apply(clean_text)
import matplotlib.pyplot as plt
import seaborn as sns
sns.countplot(df2['class name'])
plt.xlabel('class name')
<mask token>
outfit = pd.get_dummies(df2['class name'])
fit = pd.concat([df2, outfit], axis=1)
fit = fit.drop(columns=['class name'])
fit
review = fit['review text'].values
label = fit[['Fine gauge', 'Jeans', 'Pants']].values
review
label
from sklearn.model_selection import train_test_split
review_train, review_test, label_train, label_test = train_test_split(review,
    label, test_size=0.2, shuffle=True)
import numpy as np
from gensim.parsing.porter import PorterStemmer
from gensim.parsing.preprocessing import remove_stopwords
porterStemmer = PorterStemmer()
rev_train_texts = []
for text in review_train:
    removed_text = remove_stopwords(text)
    removed_text = porterStemmer.stem_sentence(removed_text)
    rev_train_texts.append(removed_text)
rev_train_texts = np.array(rev_train_texts, dtype='O')
rev_test_texts = []
for text in review_test:
    removed_text = remove_stopwords(text)
    removed_text = porterStemmer.stem_sentence(removed_text)
    rev_test_texts.append(removed_text)
rev_test_texts = np.array(rev_test_texts, dtype='O')
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
tokenizer = Tokenizer(num_words=5000, oov_token='x')
tokenizer.fit_on_texts(rev_train_texts)
tokenizer.fit_on_texts(rev_test_texts)
seq_train = tokenizer.texts_to_sequences(rev_train_texts)
seq_test = tokenizer.texts_to_sequences(rev_test_texts)
padded_train = pad_sequences(seq_train)
padded_test = pad_sequences(seq_test)
word_to_index = tokenizer.word_index
len(word_to_index)
import tensorflow as tf
model = tf.keras.Sequential([tf.keras.layers.Embedding(input_dim=5000,
    output_dim=32), tf.keras.layers.LSTM(64, dropout=0.1, recurrent_dropout
    =0.1), tf.keras.layers.Flatten(), tf.keras.layers.Dropout(0.2), tf.
    keras.layers.Dense(128, activation='relu'), tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(64, activation=
    'relu'), tf.keras.layers.Flatten(), tf.keras.layers.Dropout(0.2), tf.
    keras.layers.Dense(32, activation='relu'), tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(3, activation=
    'softmax')])
model.summary()
opt = tf.keras.optimizers.Adam(learning_rate=0.001)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['acc'])
from keras.callbacks import EarlyStopping, ReduceLROnPlateau
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=1,
    min_lr=1e-05)
early_stopping_monitor = EarlyStopping(monitor='val_acc', min_delta=0,
    patience=10, verbose=1, restore_best_weights=True)
hist = model.fit(padded_train, label_train, steps_per_epoch=25, epochs=50,
    validation_data=(padded_test, label_test), validation_steps=5, verbose=
    1, callbacks=[early_stopping_monitor, reduce_lr])
plt.plot(hist.history['acc'])
plt.plot(hist.history['val_acc'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper left')
plt.show()
plt.plot(hist.history['loss'])
plt.plot(hist.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper right')
plt.show()

# -*- coding: utf-8 -*-
"""Dicoding_Submission_Intermediate_NLP#1.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1wuIaJJQ11A_C4yVEBE7aCX0BGI86nRJA
"""

from google.colab import drive
drive.mount('/content/drive')

import pandas as pd
df = pd.read_csv('/content/drive/My Drive/Womens Clothing E-Commerce Reviews.csv')

df.head()

"""dataset https://www.kaggle.com/nicapotato/womens-ecommerce-clothing-reviews"""

len(df['Class Name'].unique())

df['Class Name'].unique()

df['Department Name'].value_counts()

df1 = df.drop(columns=['Unnamed: 0', 'Clothing ID', 'Age','Title', 'Rating', 'Recommended IND', 'Positive Feedback Count', 'Division Name',  'Department Name'])

df2 = df1.loc[(df['Class Name'] == 'Pants') | (df['Class Name'] == 'Jeans') | (df['Class Name'] == 'Fine gauge') ]

df2.head()

df2.columns = df2.columns.str.lower()
df2['review text'] = df2['review text'].astype(str)

# fungsi untuk text cleaning
import re

def clean_text(text):
    text = text.lower()
    text = re.sub(r"what's", "what is ", text)
    text = re.sub(r"\'s", " ", text)
    text = re.sub(r"\'ve", " have ", text)
    text = re.sub(r"can't", "can not ", text)
    text = re.sub(r"n't", " not ", text)
    text = re.sub(r"i'm", "i am ", text)
    text = re.sub(r"\'re", " are ", text)
    text = re.sub(r"\'d", " would ", text)
    text = re.sub(r"\'ll", " will ", text)
    text = re.sub(r"\'scuse", " excuse ", text)
    #text = re.sub('\W', ' ', text)
    #text = re.sub('\s+', ' ', text)
    text = text.strip(' ')
    return text

df2['review text'] = df2['review text'].apply(clean_text)

# Commented out IPython magic to ensure Python compatibility.
import matplotlib.pyplot as plt
# %matplotlib inline
import seaborn as sns

sns.countplot(df2['class name'])
plt.xlabel('class name')

"""Data cukup balanced sehingga cocok digunakan sebagai klasifikasi"""

outfit = pd.get_dummies(df2['class name'])
fit = pd.concat([df2, outfit], axis=1)
fit = fit.drop(columns=['class name'])

fit

review = fit['review text'].values
label = fit[['Fine gauge', 'Jeans', 'Pants']].values

review

label

from sklearn.model_selection import train_test_split
review_train, review_test, label_train, label_test = train_test_split(review, label, test_size=0.2, shuffle=True)

import numpy as np
from gensim.parsing.porter import PorterStemmer
from gensim.parsing.preprocessing import remove_stopwords

porterStemmer = PorterStemmer()

rev_train_texts = []
for text in review_train:
  removed_text = remove_stopwords(text)   
  removed_text = porterStemmer.stem_sentence(removed_text) 
  rev_train_texts.append(removed_text)

rev_train_texts = np.array(rev_train_texts, dtype='O')  

rev_test_texts = []
for text in review_test:
  removed_text = remove_stopwords(text)
  removed_text = porterStemmer.stem_sentence(removed_text)
  rev_test_texts.append(removed_text)

rev_test_texts = np.array(rev_test_texts, dtype='O')

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
 
tokenizer = Tokenizer(num_words=5000, oov_token='x')
tokenizer.fit_on_texts(rev_train_texts) 
tokenizer.fit_on_texts(rev_test_texts)
	 
seq_train = tokenizer.texts_to_sequences(rev_train_texts)
seq_test = tokenizer.texts_to_sequences(rev_test_texts)
	 
padded_train = pad_sequences(seq_train) 
padded_test = pad_sequences(seq_test)

word_to_index = tokenizer.word_index

len(word_to_index)

import tensorflow as tf

model = tf.keras.Sequential([
    tf.keras.layers.Embedding(input_dim=5000, output_dim=32),
    tf.keras.layers.LSTM(64, dropout=0.1, recurrent_dropout=0.1),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Dense(128,activation = 'relu'),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Dense(64,activation = 'relu'),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Dense(32,activation = 'relu'),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Dense(3, activation='softmax')
])

model.summary()

opt = tf.keras.optimizers.Adam(learning_rate=0.001)
model.compile(loss = 'categorical_crossentropy', optimizer = opt, metrics = ['acc'])

from keras.callbacks import EarlyStopping,ReduceLROnPlateau

reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
                              patience=1, min_lr=0.00001)

early_stopping_monitor = EarlyStopping(monitor = 'val_acc',
                          min_delta = 0,
                          patience = 10,
                          verbose = 1,
                          restore_best_weights = True)

hist = model.fit(padded_train,label_train,steps_per_epoch=25,epochs=50,validation_data=(padded_test, label_test),validation_steps=5,verbose=1, callbacks=[early_stopping_monitor,reduce_lr])

#Visualize the model accuracy
plt.plot(hist.history['acc'])
plt.plot(hist.history['val_acc'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper left')
plt.show()

#Visualize the model loss
plt.plot(hist.history['loss'])
plt.plot(hist.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper right')
plt.show()