import json as j
import pandas as pd
import re
import numpy as np
import nltk
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split 
from sklearn.feature_selection import SelectKBest, chi2

json_data = None
with open('yelp_academic_dataset_review.json') as data_file:
  '''
  lines = []
  for x in range(50000):  # I didn't choose whole dataset as it didn't fit into memory. 
    
    lines.append(data_file.readline())
  '''
  lines = data_file.readlines()  #to train only on subset of whole data comment this line and uncomment above 3 lines.
  joined_lines =  "["+ ",".join(lines) +  "]" 
  json_data = j.loads(joined_lines)
  
data = pd.DataFrame(json_data)

nltk.download('stopwords')
stemmer = SnowballStemmer('english')
words = stopwords.words('english')

data['cleaned'] = data['text'].apply(lambda x: " ".join([stemmer.stem(i) for i in re.sub("[^a-zA-Z]"," ",x).split() if i not in words]).lower())

X_train,X_test,y_train,y_test = train_test_split(data['cleaned'], data.stars, test_size = 0.2)

pipeline = Pipeline([('vect',TfidfVectorizer(ngram_range = (1,2), stop_words = "english", sublinear_tf = True)),
                     ('chi',SelectKBest(chi2, k = 10000)),
                     ('clf',LinearSVC(C = 1.0, penalty = 'l1', max_iter = 3000, dual = False))])

model = pipeline.fit(X_train,y_train)

vectorizer = model.named_steps['vect']
chi = model.named_steps['chi']
clf = model.named_steps['clf']

feature_names = vectorizer.get_feature_names()
feature_names = [feature_names[i] for i in chi.get_support(indices = True)]
feature_names = np.asarray(feature_names)

target_names = ['1','2','3','4','5']
print("top 10 keywords per class : ")
for i, label in enumerate(target_names):
    top10 = np.argsort(clf.coef_[i])[-10:]
    print("%s %s"%(label," ".join(feature_names[top10])))

print("accuracy score :" + str(model.score(X_test,y_test)))

print(model.predict(['That was an awesome place. great food']))

542ff78d2a1d1181b04f939da5ae667cdd95d906
<mask token>

<mask token>
with open('yelp_academic_dataset_review.json') as data_file:
    """
  lines = []
  for x in range(50000):  # I didn't choose whole dataset as it didn't fit into memory. 
    
    lines.append(data_file.readline())
  """
    lines = data_file.readlines()
    joined_lines = '[' + ','.join(lines) + ']'
    json_data = j.loads(joined_lines)
<mask token>
nltk.download('stopwords')
<mask token>
print('top 10 keywords per class : ')
for i, label in enumerate(target_names):
    top10 = np.argsort(clf.coef_[i])[-10:]
    print('%s %s' % (label, ' '.join(feature_names[top10])))
print('accuracy score :' + str(model.score(X_test, y_test)))
print(model.predict(['That was an awesome place. great food']))

<mask token>
json_data = None
with open('yelp_academic_dataset_review.json') as data_file:
    """
  lines = []
  for x in range(50000):  # I didn't choose whole dataset as it didn't fit into memory. 
    
    lines.append(data_file.readline())
  """
    lines = data_file.readlines()
    joined_lines = '[' + ','.join(lines) + ']'
    json_data = j.loads(joined_lines)
data = pd.DataFrame(json_data)
nltk.download('stopwords')
stemmer = SnowballStemmer('english')
words = stopwords.words('english')
data['cleaned'] = data['text'].apply(lambda x: ' '.join([stemmer.stem(i) for
    i in re.sub('[^a-zA-Z]', ' ', x).split() if i not in words]).lower())
X_train, X_test, y_train, y_test = train_test_split(data['cleaned'], data.
    stars, test_size=0.2)
pipeline = Pipeline([('vect', TfidfVectorizer(ngram_range=(1, 2),
    stop_words='english', sublinear_tf=True)), ('chi', SelectKBest(chi2, k=
    10000)), ('clf', LinearSVC(C=1.0, penalty='l1', max_iter=3000, dual=
    False))])
model = pipeline.fit(X_train, y_train)
vectorizer = model.named_steps['vect']
chi = model.named_steps['chi']
clf = model.named_steps['clf']
feature_names = vectorizer.get_feature_names()
feature_names = [feature_names[i] for i in chi.get_support(indices=True)]
feature_names = np.asarray(feature_names)
target_names = ['1', '2', '3', '4', '5']
print('top 10 keywords per class : ')
for i, label in enumerate(target_names):
    top10 = np.argsort(clf.coef_[i])[-10:]
    print('%s %s' % (label, ' '.join(feature_names[top10])))
print('accuracy score :' + str(model.score(X_test, y_test)))
print(model.predict(['That was an awesome place. great food']))

import json as j
import pandas as pd
import re
import numpy as np
import nltk
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.feature_selection import SelectKBest, chi2
json_data = None
with open('yelp_academic_dataset_review.json') as data_file:
    """
  lines = []
  for x in range(50000):  # I didn't choose whole dataset as it didn't fit into memory. 
    
    lines.append(data_file.readline())
  """
    lines = data_file.readlines()
    joined_lines = '[' + ','.join(lines) + ']'
    json_data = j.loads(joined_lines)
data = pd.DataFrame(json_data)
nltk.download('stopwords')
stemmer = SnowballStemmer('english')
words = stopwords.words('english')
data['cleaned'] = data['text'].apply(lambda x: ' '.join([stemmer.stem(i) for
    i in re.sub('[^a-zA-Z]', ' ', x).split() if i not in words]).lower())
X_train, X_test, y_train, y_test = train_test_split(data['cleaned'], data.
    stars, test_size=0.2)
pipeline = Pipeline([('vect', TfidfVectorizer(ngram_range=(1, 2),
    stop_words='english', sublinear_tf=True)), ('chi', SelectKBest(chi2, k=
    10000)), ('clf', LinearSVC(C=1.0, penalty='l1', max_iter=3000, dual=
    False))])
model = pipeline.fit(X_train, y_train)
vectorizer = model.named_steps['vect']
chi = model.named_steps['chi']
clf = model.named_steps['clf']
feature_names = vectorizer.get_feature_names()
feature_names = [feature_names[i] for i in chi.get_support(indices=True)]
feature_names = np.asarray(feature_names)
target_names = ['1', '2', '3', '4', '5']
print('top 10 keywords per class : ')
for i, label in enumerate(target_names):
    top10 = np.argsort(clf.coef_[i])[-10:]
    print('%s %s' % (label, ' '.join(feature_names[top10])))
print('accuracy score :' + str(model.score(X_test, y_test)))
print(model.predict(['That was an awesome place. great food']))

import json as j
import pandas as pd
import re
import numpy as np
import nltk
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split 
from sklearn.feature_selection import SelectKBest, chi2

json_data = None
with open('yelp_academic_dataset_review.json') as data_file:
  '''
  lines = []
  for x in range(50000):  # I didn't choose whole dataset as it didn't fit into memory. 
    
    lines.append(data_file.readline())
  '''
  lines = data_file.readlines()  #to train only on subset of whole data comment this line and uncomment above 3 lines.
  joined_lines =  "["+ ",".join(lines) +  "]" 
  json_data = j.loads(joined_lines)
  
data = pd.DataFrame(json_data)

nltk.download('stopwords')
stemmer = SnowballStemmer('english')
words = stopwords.words('english')

data['cleaned'] = data['text'].apply(lambda x: " ".join([stemmer.stem(i) for i in re.sub("[^a-zA-Z]"," ",x).split() if i not in words]).lower())

X_train,X_test,y_train,y_test = train_test_split(data['cleaned'], data.stars, test_size = 0.2)

pipeline = Pipeline([('vect',TfidfVectorizer(ngram_range = (1,2), stop_words = "english", sublinear_tf = True)),
                     ('chi',SelectKBest(chi2, k = 10000)),
                     ('clf',LinearSVC(C = 1.0, penalty = 'l1', max_iter = 3000, dual = False))])

model = pipeline.fit(X_train,y_train)

vectorizer = model.named_steps['vect']
chi = model.named_steps['chi']
clf = model.named_steps['clf']

feature_names = vectorizer.get_feature_names()
feature_names = [feature_names[i] for i in chi.get_support(indices = True)]
feature_names = np.asarray(feature_names)

target_names = ['1','2','3','4','5']
print("top 10 keywords per class : ")
for i, label in enumerate(target_names):
    top10 = np.argsort(clf.coef_[i])[-10:]
    print("%s %s"%(label," ".join(feature_names[top10])))

print("accuracy score :" + str(model.score(X_test,y_test)))

print(model.predict(['That was an awesome place. great food']))
