import json
from pprint import pprint
import preprocessor as pre
from nltk.tokenize import RegexpTokenizer

with open('/data/CORPUS/topics/topics_2016.json') as f:
    data = json.load(f)
print(len(data))
left_data=[]
tokenizer = RegexpTokenizer(r'\w+')
for topic in data:
    text=topic['title']
    text=pre.preProcessText(text)
    tokens=tokenizer.tokenize(text)
    stems=pre.stemTokens(tokens)
    text=" ".join([stem for stem in stems])
    left_data+= [{"title":text,"topid":topic['topid']}]
print(len(left_data))
print(left_data[:10])
with open('/data/CORPUS/left/left_2016.json', 'w') as outfile:
    json.dump(left_data, outfile)
2bd24fc9b0cc7983d05f61a7d536d2ebd6fe0c17
<mask token>

<mask token>
with open('/data/CORPUS/topics/topics_2016.json') as f:
    data = json.load(f)
print(len(data))
<mask token>
for topic in data:
    text = topic['title']
    text = pre.preProcessText(text)
    tokens = tokenizer.tokenize(text)
    stems = pre.stemTokens(tokens)
    text = ' '.join([stem for stem in stems])
    left_data += [{'title': text, 'topid': topic['topid']}]
print(len(left_data))
print(left_data[:10])
with open('/data/CORPUS/left/left_2016.json', 'w') as outfile:
    json.dump(left_data, outfile)

<mask token>
with open('/data/CORPUS/topics/topics_2016.json') as f:
    data = json.load(f)
print(len(data))
left_data = []
tokenizer = RegexpTokenizer('\\w+')
for topic in data:
    text = topic['title']
    text = pre.preProcessText(text)
    tokens = tokenizer.tokenize(text)
    stems = pre.stemTokens(tokens)
    text = ' '.join([stem for stem in stems])
    left_data += [{'title': text, 'topid': topic['topid']}]
print(len(left_data))
print(left_data[:10])
with open('/data/CORPUS/left/left_2016.json', 'w') as outfile:
    json.dump(left_data, outfile)

import json
from pprint import pprint
import preprocessor as pre
from nltk.tokenize import RegexpTokenizer
with open('/data/CORPUS/topics/topics_2016.json') as f:
    data = json.load(f)
print(len(data))
left_data = []
tokenizer = RegexpTokenizer('\\w+')
for topic in data:
    text = topic['title']
    text = pre.preProcessText(text)
    tokens = tokenizer.tokenize(text)
    stems = pre.stemTokens(tokens)
    text = ' '.join([stem for stem in stems])
    left_data += [{'title': text, 'topid': topic['topid']}]
print(len(left_data))
print(left_data[:10])
with open('/data/CORPUS/left/left_2016.json', 'w') as outfile:
    json.dump(left_data, outfile)

import json
from pprint import pprint
import preprocessor as pre
from nltk.tokenize import RegexpTokenizer

with open('/data/CORPUS/topics/topics_2016.json') as f:
    data = json.load(f)
print(len(data))
left_data=[]
tokenizer = RegexpTokenizer(r'\w+')
for topic in data:
    text=topic['title']
    text=pre.preProcessText(text)
    tokens=tokenizer.tokenize(text)
    stems=pre.stemTokens(tokens)
    text=" ".join([stem for stem in stems])
    left_data+= [{"title":text,"topid":topic['topid']}]
print(len(left_data))
print(left_data[:10])
with open('/data/CORPUS/left/left_2016.json', 'w') as outfile:
    json.dump(left_data, outfile)