import urllib3
import urllib
import json
import pandas as pd
import bs4
import re
import nltk
import datetime
from nltk.corpus import words
from nltk.corpus import stopwords
import math
from pycorenlp import StanfordCoreNLP
from fuzzywuzzy import fuzz
from fuzzywuzzy import process
from nltk.stem.snowball import SnowballStemmer


stemmer = SnowballStemmer("spanish")
news_api_keys = ['a0849f217c5d4628a7350ce96868c85d', '']


def make_soup(myurl):
    pm = urllib3.PoolManager()
    html = pm.urlopen(url = myurl, method = 'GET', redirect= False).data
    return bs4.BeautifulSoup(html, 'html5lib')

def get_text_news(url, tag = 'p'):
    soup = make_soup(url)
    text = ' '.join([t.text.encode('utf-8', 'ignore').decode('utf-8') for t in soup.find_all('p')])
    #print(url)
    return text

def valid_url(url):
    a=urllib.parse.urlsplit(url)
    valid = urllib.parse.urlunparse((a.scheme, a.netloc, 
                                     urllib.parse.quote(a.path.encode('utf-8')), '','', ''))
    return valid

def get_entities(text, nlp=nlp):
    return nlp.annotate(text, properties={'annotators': 'ner', 'outputFormat': 'json'})


def get_entity_tup(annotator_dict):
    ners = []
    for sentence in annotator_dict['sentences']:
        for entity in sentence['entitymentions']:
            ners.append((entity['ner'],entity['text']))
    return list(set(ners))


def is_relevant(tup_list, relevant_words):
    relevant_tags = ['CAUSE_OF_DEATH', 'CRIMINAL_CHARGE']
    entity_list = [tup[1] for tup in tup_list if tup[0] in relevant_tags]
    to_compare = set([stemmer.stem(entity.lower()) for entity in entity_list])
    relevant_words = set(relevant_words)
    intersect = list(relevant_words.intersection(to_compare))
    if len(intersect) > 0:
        return 1
    else:
        return 0


def build_keywords_list(cities_list, crimes_list):
	crimes_string = ' OR '.join(crimes_list)
	return [city+' AND '+'('+crimes_string+')' for city in cities_list] 



def build_url(keywords_list, date_from, date_to, languaje = 'es', key=news_api_key):
	url = 'https://newsapi.org/v2/everything?q={}&from={}&to={}&language={}&sortBy=relevancy&pageSize=100&apiKey={}'
	keywords = urllib.parse.quote(keywords_list)
	myurl = url.format(keywords, date_from, date_to, languaje, key)
	return myurl

def get_news_json(myurl):
	http = urllib3.PoolManager()
	r = http.request('GET', myurl)
	return json.loads(r.data.decode('utf-8'))

def get_values(dictionary, key):
	return [article[key] for article in dictionary.values() if article['unique']==1]

def text_from_url(url, content):
	soup = make_soup(url)
	all_text = [t.text for t in soup.find_all(True) if content in t.text]
	lens = [len(i) for i in all_text]
	index_text = lens.index(max(lens))
	return all_text[index_text]

def news_dict_of_dict(responses):
    '''Creates a dictionary of dictionaries where the key
    is a unique ID for each news, and the value is a dictionary with the contents
    of the news
    input: responses (list) 
           it's the output from the requests. Each element of this list is a dictionary with one or many
           news
    output: a new dictionary with an id for each news
    
           '''
    news_collection = {}
    for response in responses:
        if 'articles' in response.keys():
            for article in response['articles']:
                news_collection[article['url']] = article
    return news_collection
    
def similarity_score(dict1,dict2):
    '''Returns a similarity score to test if two strings
    in two 'different' news are in reality the same
    inputs: dict1 and dict2: dictionaries
    output: score (int)'''
    score_1 = fuzz.token_set_ratio(dict1['title'], dict2['title'])
    score_2 = fuzz.token_set_ratio(dict1['description'], dict2['description'])
    score_3 = fuzz.token_set_ratio(dict1['content'], dict2['content'])
    return 0.4*score_1 + 0.3*score_2 + 0.3*score_3

# def compare_similarity(d):
#     start = 0
#     size = len(d)
#     t = 80
#     for k, v in d.items():
#         for key in range(start + 1, size):
#             if d[key]['unique'] == 1:
#                 s = similarity_score(d[start], d[key])
#                 if s > t:
#                     d[key]['unique'] = 0
#         start += 1

def compare_similarity(d):

    size = len(d)
    t = 80
<<<<<<< HEAD
    for start in range(:size):
    	temp = []
	    for key in range(start + 1, size):
	        if d[key]['unique'] == 1:
	            s = similarity_score(d[start], d[key])
	            if s > t:
	            	temp.append(key)
	    for index in temp:
	    	d[index]['unique'] = 0
        
=======
    for start in range(size):
        temp = []
        for key in range(start + 1, size):
            if d[key]['unique'] == 1:
                s = similarity_score(d[start], d[key])
                if s > t:
                    temp.append(key)
        for index in temp:
            d[index]['unique'] = 0
from nltk.corpus import stopwords

def clean_sentence(sentence):
    '''
    Removes unknown characters or unknown words, change capital to lower letters and remove
    english stop words
    Inputs:
    sentence (string): a sting to be cleaned
    Returns: a string
    '''
    new = ''
    for l in sentence:
        if re.match('[a-zA-Z0-9_\s]',l):
            new += l

    tokens = nltk.word_tokenize(new)
    tokens = [t.lower() for t in tokens]

    new_tokens = []
    imp_words = set(stopwords.words('spanish'))
    for t in tokens:
        if t in imp_words:
            new_tokens.append(t)

    return ' '.join(new_tokens)
        
>>>>>>> upstream/master

5569235eb1df5041c90936f37df980daaa3ad42f
import urllib3
import urllib
import json
import pandas as pd
import bs4
import re
import nltk
import datetime
from nltk.corpus import words
from nltk.corpus import stopwords
import math
from pycorenlp import StanfordCoreNLP
from fuzzywuzzy import fuzz
from fuzzywuzzy import process
from nltk.stem.snowball import SnowballStemmer


stemmer = SnowballStemmer("spanish")
news_api_keys = ['a0849f217c5d4628a7350ce96868c85d', '']


def make_soup(myurl):
    pm = urllib3.PoolManager()
    html = pm.urlopen(url = myurl, method = 'GET', redirect= False).data
    return bs4.BeautifulSoup(html, 'html5lib')

def get_text_news(url, tag = 'p'):
    soup = make_soup(url)
    text = ' '.join([t.text.encode('utf-8', 'ignore').decode('utf-8') for t in soup.find_all('p')])
    #print(url)
    return text

def valid_url(url):
    a=urllib.parse.urlsplit(url)
    valid = urllib.parse.urlunparse((a.scheme, a.netloc, 
                                     urllib.parse.quote(a.path.encode('utf-8')), '','', ''))
    return valid

def get_entities(text, nlp=nlp):
    return nlp.annotate(text, properties={'annotators': 'ner', 'outputFormat': 'json'})


def get_entity_tup(annotator_dict):
    ners = []
    for sentence in annotator_dict['sentences']:
        for entity in sentence['entitymentions']:
            ners.append((entity['ner'],entity['text']))
    return list(set(ners))


def is_relevant(tup_list, relevant_words):
    relevant_tags = ['CAUSE_OF_DEATH', 'CRIMINAL_CHARGE']
    entity_list = [tup[1] for tup in tup_list if tup[0] in relevant_tags]
    to_compare = set([stemmer.stem(entity.lower()) for entity in entity_list])
    relevant_words = set(relevant_words)
    intersect = list(relevant_words.intersection(to_compare))
    if len(intersect) > 0:
        return 1
    else:
        return 0


def build_keywords_list(cities_list, crimes_list):
	crimes_string = ' OR '.join(crimes_list)
	return [city+' AND '+'('+crimes_string+')' for city in cities_list] 



def build_url(keywords_list, date_from, date_to, languaje = 'es', key=news_api_key):
	url = 'https://newsapi.org/v2/everything?q={}&from={}&to={}&language={}&sortBy=relevancy&pageSize=100&apiKey={}'
	keywords = urllib.parse.quote(keywords_list)
	myurl = url.format(keywords, date_from, date_to, languaje, key)
	return myurl

def get_news_json(myurl):
	http = urllib3.PoolManager()
	r = http.request('GET', myurl)
	return json.loads(r.data.decode('utf-8'))

def get_values(dictionary, key):
	return [article[key] for article in dictionary.values() if article['unique']==1]

def text_from_url(url, content):
	soup = make_soup(url)
	all_text = [t.text for t in soup.find_all(True) if content in t.text]
	lens = [len(i) for i in all_text]
	index_text = lens.index(max(lens))
	return all_text[index_text]

def news_dict_of_dict(responses):
    '''Creates a dictionary of dictionaries where the key
    is a unique ID for each news, and the value is a dictionary with the contents
    of the news
    input: responses (list) 
           it's the output from the requests. Each element of this list is a dictionary with one or many
           news
    output: a new dictionary with an id for each news
    
           '''
    news_collection = {}
    for response in responses:
        if 'articles' in response.keys():
            for article in response['articles']:
                news_collection[article['url']] = article
    return news_collection
    
def similarity_score(dict1,dict2):
    '''Returns a similarity score to test if two strings
    in two 'different' news are in reality the same
    inputs: dict1 and dict2: dictionaries
    output: score (int)'''
    score_1 = fuzz.token_set_ratio(dict1['title'], dict2['title'])
    score_2 = fuzz.token_set_ratio(dict1['description'], dict2['description'])
    score_3 = fuzz.token_set_ratio(dict1['content'], dict2['content'])
    return 0.4*score_1 + 0.3*score_2 + 0.3*score_3

# def compare_similarity(d):
#     start = 0
#     size = len(d)
#     t = 80
#     for k, v in d.items():
#         for key in range(start + 1, size):
#             if d[key]['unique'] == 1:
#                 s = similarity_score(d[start], d[key])
#                 if s > t:
#                     d[key]['unique'] = 0
#         start += 1

def compare_similarity(d):

    size = len(d)
    t = 80
<<<<<<< HEAD
    for start in range(:size):
    	temp = []
	    for key in range(start + 1, size):
	        if d[key]['unique'] == 1:
	            s = similarity_score(d[start], d[key])
	            if s > t:
	            	temp.append(key)
	    for index in temp:
	    	d[index]['unique'] = 0
        
=======
    for start in range(size):
        temp = []
        for key in range(start + 1, size):
            if d[key]['unique'] == 1:
                s = similarity_score(d[start], d[key])
                if s > t:
                    temp.append(key)
        for index in temp:
            d[index]['unique'] = 0
from nltk.corpus import stopwords

def clean_sentence(sentence):
    '''
    Removes unknown characters or unknown words, change capital to lower letters and remove
    english stop words
    Inputs:
    sentence (string): a sting to be cleaned
    Returns: a string
    '''
    new = ''
    for l in sentence:
        if re.match('[a-zA-Z0-9_\s]',l):
            new += l

    tokens = nltk.word_tokenize(new)
    tokens = [t.lower() for t in tokens]

    new_tokens = []
    imp_words = set(stopwords.words('spanish'))
    for t in tokens:
        if t in imp_words:
            new_tokens.append(t)

    return ' '.join(new_tokens)
        
>>>>>>> upstream/master
