#!/usr/bin/env python
# coding: utf-8

# #### Create a NLP pipeline to 'clean' Reviews Data
# - Load input File and read reviews
# - Tokenise
# - Remove stopwrods
# - perform Stemming
# - Write cleaned data to output file

# In[17]:


import sys
from nltk.tokenize import RegexpTokenizer
from nltk.stem.snowball import EnglishStemmer
from nltk.corpus import stopwords


# In[18]:


tokenizer = RegexpTokenizer(r'\w+')
en_stopwords = set(stopwords.words('english'))
ps = EnglishStemmer()


# In[19]:


def getcleanRev(review):

    review = review.lower()
    # review = review.replace("<br /><br />"," ")

    # RegexpTokenizer
    tokens = tokenizer.tokenize(review)
    new_tokens = [ token for token in tokens if token not in en_stopwords]
    stemmed_tokens = [ps.stem(token) for token in new_tokens]

    clean_review = ' '.join(stemmed_tokens)

    return clean_review


# In[20]:


sample_text = "mature intelligent and highly charged melodrama unbelivebly filmed in China in 1948. wei wei's stunning performance as the catylast in a love triangle is simply stunning if you have the oppurunity to see this magnificent film take it,pos"


# In[21]:


print(getcleanRev(sample_text))

c8ea98b568c5c00988a6e4f906515ea63c905a99
<mask token>


def getcleanRev(review):
    review = review.lower()
    tokens = tokenizer.tokenize(review)
    new_tokens = [token for token in tokens if token not in en_stopwords]
    stemmed_tokens = [ps.stem(token) for token in new_tokens]
    clean_review = ' '.join(stemmed_tokens)
    return clean_review


<mask token>

<mask token>


def getcleanRev(review):
    review = review.lower()
    tokens = tokenizer.tokenize(review)
    new_tokens = [token for token in tokens if token not in en_stopwords]
    stemmed_tokens = [ps.stem(token) for token in new_tokens]
    clean_review = ' '.join(stemmed_tokens)
    return clean_review


<mask token>
print(getcleanRev(sample_text))

<mask token>
tokenizer = RegexpTokenizer('\\w+')
en_stopwords = set(stopwords.words('english'))
ps = EnglishStemmer()


def getcleanRev(review):
    review = review.lower()
    tokens = tokenizer.tokenize(review)
    new_tokens = [token for token in tokens if token not in en_stopwords]
    stemmed_tokens = [ps.stem(token) for token in new_tokens]
    clean_review = ' '.join(stemmed_tokens)
    return clean_review


sample_text = (
    "mature intelligent and highly charged melodrama unbelivebly filmed in China in 1948. wei wei's stunning performance as the catylast in a love triangle is simply stunning if you have the oppurunity to see this magnificent film take it,pos"
    )
print(getcleanRev(sample_text))

import sys
from nltk.tokenize import RegexpTokenizer
from nltk.stem.snowball import EnglishStemmer
from nltk.corpus import stopwords
tokenizer = RegexpTokenizer('\\w+')
en_stopwords = set(stopwords.words('english'))
ps = EnglishStemmer()


def getcleanRev(review):
    review = review.lower()
    tokens = tokenizer.tokenize(review)
    new_tokens = [token for token in tokens if token not in en_stopwords]
    stemmed_tokens = [ps.stem(token) for token in new_tokens]
    clean_review = ' '.join(stemmed_tokens)
    return clean_review


sample_text = (
    "mature intelligent and highly charged melodrama unbelivebly filmed in China in 1948. wei wei's stunning performance as the catylast in a love triangle is simply stunning if you have the oppurunity to see this magnificent film take it,pos"
    )
print(getcleanRev(sample_text))

#!/usr/bin/env python
# coding: utf-8

# #### Create a NLP pipeline to 'clean' Reviews Data
# - Load input File and read reviews
# - Tokenise
# - Remove stopwrods
# - perform Stemming
# - Write cleaned data to output file

# In[17]:


import sys
from nltk.tokenize import RegexpTokenizer
from nltk.stem.snowball import EnglishStemmer
from nltk.corpus import stopwords


# In[18]:


tokenizer = RegexpTokenizer(r'\w+')
en_stopwords = set(stopwords.words('english'))
ps = EnglishStemmer()


# In[19]:


def getcleanRev(review):

    review = review.lower()
    # review = review.replace("<br /><br />"," ")

    # RegexpTokenizer
    tokens = tokenizer.tokenize(review)
    new_tokens = [ token for token in tokens if token not in en_stopwords]
    stemmed_tokens = [ps.stem(token) for token in new_tokens]

    clean_review = ' '.join(stemmed_tokens)

    return clean_review


# In[20]:


sample_text = "mature intelligent and highly charged melodrama unbelivebly filmed in China in 1948. wei wei's stunning performance as the catylast in a love triangle is simply stunning if you have the oppurunity to see this magnificent film take it,pos"


# In[21]:


print(getcleanRev(sample_text))
