import gensim
import warnings
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')
from gensim.models.keyedvectors import KeyedVectors
from numpy import dot
from numpy.linalg import norm
import pandas as pd
import ast, heapq

def cosine_sim(a,b):
    return dot(a, b)/(norm(a)*norm(b))

def f(wlist, model, num):
    wlist = [word for word in wlist if word in model.wv.vocab]
    lenwlist=len(wlist)
    if lenwlist == 1:
        return wlist
    elif lenwlist == 0:
        return []
    avrsim=[]
    #compute cosine similarity between each word in wlist with the other words in wlist  
    for i in range(lenwlist):
        word=wlist[i]
        totalsim=0
        wordembed=model[word] 
        for j in range(lenwlist):
            if i!=j:
                word2embed=model[wlist[j]] 
                totalsim+=cosine_sim(wordembed, word2embed)
        avrsim.append(totalsim/ (lenwlist-1))   
    t = heapq.nlargest(num,avrsim)
    return [wlist[avrsim.index(i)] for i in t]

def grouping(topics):
    res = []
    for _l in topics:
        res.extend(_l)
    return res

model_path_en = "./GoogleNews-vectors-negative300.bin"
model_en = KeyedVectors.load_word2vec_format(model_path_en, binary=True)
model_path_vi = "./baomoi.window2.vn.model.bin"
model_vi = KeyedVectors.load_word2vec_format(model_path_vi, binary=True)

df = pd.read_csv("result.csv", converters={"topics_from_en": ast.literal_eval, "topics_from_vi": ast.literal_eval})
vi = df.groupby("keyword").apply(lambda x: grouping(x.topics_from_vi.tolist())).reset_index(level=0)
df["res_en"] = df["topics_from_en"].apply(lambda x: f(x, model_en, 2))
en = df.groupby("keyword").apply(lambda x: grouping(x.res_en.tolist())).reset_index(level=0)
df = pd.merge(en, vi, on="keyword")
df.columns = ["keyword", "en", "vi"]
df["en"] = df["en"].apply(lambda x: list(set(x)))
df["vi"] = df["vi"].apply(lambda x: f(x, model_vi, 3))
df["vi"] = df["vi"].apply(lambda x: list(set(x)))
df.to_csv("res_topics.csv", index=False)

df = pd.read_csv("result_web_title.csv", converters={"topics_from_en": ast.literal_eval, "topics_from_vi": ast.literal_eval})
vi = df.groupby("keyword").apply(lambda x: grouping(x.topics_from_vi.tolist())).reset_index(level=0)
df["res_en"] = df["topics_from_en"].apply(lambda x: f(x, model_en, 2))
en = df.groupby("keyword").apply(lambda x: grouping(x.res_en.tolist())).reset_index(level=0)
df = pd.merge(en, vi, on="keyword")
df.columns = ["keyword", "en", "vi"]
df["en"] = df["en"].apply(lambda x: list(set(x)))
df["vi"] = df["vi"].apply(lambda x: f(x, model_vi, 3))
df["vi"] = df["vi"].apply(lambda x: list(set(x)))
df.to_csv("res_topics_web_title.csv", index=False)
755d938aea3a1e53d73a1aa10520af9ce319fa0f
<mask token>


def cosine_sim(a, b):
    return dot(a, b) / (norm(a) * norm(b))


def f(wlist, model, num):
    wlist = [word for word in wlist if word in model.wv.vocab]
    lenwlist = len(wlist)
    if lenwlist == 1:
        return wlist
    elif lenwlist == 0:
        return []
    avrsim = []
    for i in range(lenwlist):
        word = wlist[i]
        totalsim = 0
        wordembed = model[word]
        for j in range(lenwlist):
            if i != j:
                word2embed = model[wlist[j]]
                totalsim += cosine_sim(wordembed, word2embed)
        avrsim.append(totalsim / (lenwlist - 1))
    t = heapq.nlargest(num, avrsim)
    return [wlist[avrsim.index(i)] for i in t]


def grouping(topics):
    res = []
    for _l in topics:
        res.extend(_l)
    return res


<mask token>

<mask token>
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')
<mask token>


def cosine_sim(a, b):
    return dot(a, b) / (norm(a) * norm(b))


def f(wlist, model, num):
    wlist = [word for word in wlist if word in model.wv.vocab]
    lenwlist = len(wlist)
    if lenwlist == 1:
        return wlist
    elif lenwlist == 0:
        return []
    avrsim = []
    for i in range(lenwlist):
        word = wlist[i]
        totalsim = 0
        wordembed = model[word]
        for j in range(lenwlist):
            if i != j:
                word2embed = model[wlist[j]]
                totalsim += cosine_sim(wordembed, word2embed)
        avrsim.append(totalsim / (lenwlist - 1))
    t = heapq.nlargest(num, avrsim)
    return [wlist[avrsim.index(i)] for i in t]


def grouping(topics):
    res = []
    for _l in topics:
        res.extend(_l)
    return res


<mask token>
df.to_csv('res_topics.csv', index=False)
<mask token>
df.to_csv('res_topics_web_title.csv', index=False)

<mask token>
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')
<mask token>


def cosine_sim(a, b):
    return dot(a, b) / (norm(a) * norm(b))


def f(wlist, model, num):
    wlist = [word for word in wlist if word in model.wv.vocab]
    lenwlist = len(wlist)
    if lenwlist == 1:
        return wlist
    elif lenwlist == 0:
        return []
    avrsim = []
    for i in range(lenwlist):
        word = wlist[i]
        totalsim = 0
        wordembed = model[word]
        for j in range(lenwlist):
            if i != j:
                word2embed = model[wlist[j]]
                totalsim += cosine_sim(wordembed, word2embed)
        avrsim.append(totalsim / (lenwlist - 1))
    t = heapq.nlargest(num, avrsim)
    return [wlist[avrsim.index(i)] for i in t]


def grouping(topics):
    res = []
    for _l in topics:
        res.extend(_l)
    return res


model_path_en = './GoogleNews-vectors-negative300.bin'
model_en = KeyedVectors.load_word2vec_format(model_path_en, binary=True)
model_path_vi = './baomoi.window2.vn.model.bin'
model_vi = KeyedVectors.load_word2vec_format(model_path_vi, binary=True)
df = pd.read_csv('result.csv', converters={'topics_from_en': ast.
    literal_eval, 'topics_from_vi': ast.literal_eval})
vi = df.groupby('keyword').apply(lambda x: grouping(x.topics_from_vi.tolist())
    ).reset_index(level=0)
df['res_en'] = df['topics_from_en'].apply(lambda x: f(x, model_en, 2))
en = df.groupby('keyword').apply(lambda x: grouping(x.res_en.tolist())
    ).reset_index(level=0)
df = pd.merge(en, vi, on='keyword')
df.columns = ['keyword', 'en', 'vi']
df['en'] = df['en'].apply(lambda x: list(set(x)))
df['vi'] = df['vi'].apply(lambda x: f(x, model_vi, 3))
df['vi'] = df['vi'].apply(lambda x: list(set(x)))
df.to_csv('res_topics.csv', index=False)
df = pd.read_csv('result_web_title.csv', converters={'topics_from_en': ast.
    literal_eval, 'topics_from_vi': ast.literal_eval})
vi = df.groupby('keyword').apply(lambda x: grouping(x.topics_from_vi.tolist())
    ).reset_index(level=0)
df['res_en'] = df['topics_from_en'].apply(lambda x: f(x, model_en, 2))
en = df.groupby('keyword').apply(lambda x: grouping(x.res_en.tolist())
    ).reset_index(level=0)
df = pd.merge(en, vi, on='keyword')
df.columns = ['keyword', 'en', 'vi']
df['en'] = df['en'].apply(lambda x: list(set(x)))
df['vi'] = df['vi'].apply(lambda x: f(x, model_vi, 3))
df['vi'] = df['vi'].apply(lambda x: list(set(x)))
df.to_csv('res_topics_web_title.csv', index=False)

import gensim
import warnings
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')
from gensim.models.keyedvectors import KeyedVectors
from numpy import dot
from numpy.linalg import norm
import pandas as pd
import ast, heapq


def cosine_sim(a, b):
    return dot(a, b) / (norm(a) * norm(b))


def f(wlist, model, num):
    wlist = [word for word in wlist if word in model.wv.vocab]
    lenwlist = len(wlist)
    if lenwlist == 1:
        return wlist
    elif lenwlist == 0:
        return []
    avrsim = []
    for i in range(lenwlist):
        word = wlist[i]
        totalsim = 0
        wordembed = model[word]
        for j in range(lenwlist):
            if i != j:
                word2embed = model[wlist[j]]
                totalsim += cosine_sim(wordembed, word2embed)
        avrsim.append(totalsim / (lenwlist - 1))
    t = heapq.nlargest(num, avrsim)
    return [wlist[avrsim.index(i)] for i in t]


def grouping(topics):
    res = []
    for _l in topics:
        res.extend(_l)
    return res


model_path_en = './GoogleNews-vectors-negative300.bin'
model_en = KeyedVectors.load_word2vec_format(model_path_en, binary=True)
model_path_vi = './baomoi.window2.vn.model.bin'
model_vi = KeyedVectors.load_word2vec_format(model_path_vi, binary=True)
df = pd.read_csv('result.csv', converters={'topics_from_en': ast.
    literal_eval, 'topics_from_vi': ast.literal_eval})
vi = df.groupby('keyword').apply(lambda x: grouping(x.topics_from_vi.tolist())
    ).reset_index(level=0)
df['res_en'] = df['topics_from_en'].apply(lambda x: f(x, model_en, 2))
en = df.groupby('keyword').apply(lambda x: grouping(x.res_en.tolist())
    ).reset_index(level=0)
df = pd.merge(en, vi, on='keyword')
df.columns = ['keyword', 'en', 'vi']
df['en'] = df['en'].apply(lambda x: list(set(x)))
df['vi'] = df['vi'].apply(lambda x: f(x, model_vi, 3))
df['vi'] = df['vi'].apply(lambda x: list(set(x)))
df.to_csv('res_topics.csv', index=False)
df = pd.read_csv('result_web_title.csv', converters={'topics_from_en': ast.
    literal_eval, 'topics_from_vi': ast.literal_eval})
vi = df.groupby('keyword').apply(lambda x: grouping(x.topics_from_vi.tolist())
    ).reset_index(level=0)
df['res_en'] = df['topics_from_en'].apply(lambda x: f(x, model_en, 2))
en = df.groupby('keyword').apply(lambda x: grouping(x.res_en.tolist())
    ).reset_index(level=0)
df = pd.merge(en, vi, on='keyword')
df.columns = ['keyword', 'en', 'vi']
df['en'] = df['en'].apply(lambda x: list(set(x)))
df['vi'] = df['vi'].apply(lambda x: f(x, model_vi, 3))
df['vi'] = df['vi'].apply(lambda x: list(set(x)))
df.to_csv('res_topics_web_title.csv', index=False)

import gensim
import warnings
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')
from gensim.models.keyedvectors import KeyedVectors
from numpy import dot
from numpy.linalg import norm
import pandas as pd
import ast, heapq

def cosine_sim(a,b):
    return dot(a, b)/(norm(a)*norm(b))

def f(wlist, model, num):
    wlist = [word for word in wlist if word in model.wv.vocab]
    lenwlist=len(wlist)
    if lenwlist == 1:
        return wlist
    elif lenwlist == 0:
        return []
    avrsim=[]
    #compute cosine similarity between each word in wlist with the other words in wlist  
    for i in range(lenwlist):
        word=wlist[i]
        totalsim=0
        wordembed=model[word] 
        for j in range(lenwlist):
            if i!=j:
                word2embed=model[wlist[j]] 
                totalsim+=cosine_sim(wordembed, word2embed)
        avrsim.append(totalsim/ (lenwlist-1))   
    t = heapq.nlargest(num,avrsim)
    return [wlist[avrsim.index(i)] for i in t]

def grouping(topics):
    res = []
    for _l in topics:
        res.extend(_l)
    return res

model_path_en = "./GoogleNews-vectors-negative300.bin"
model_en = KeyedVectors.load_word2vec_format(model_path_en, binary=True)
model_path_vi = "./baomoi.window2.vn.model.bin"
model_vi = KeyedVectors.load_word2vec_format(model_path_vi, binary=True)

df = pd.read_csv("result.csv", converters={"topics_from_en": ast.literal_eval, "topics_from_vi": ast.literal_eval})
vi = df.groupby("keyword").apply(lambda x: grouping(x.topics_from_vi.tolist())).reset_index(level=0)
df["res_en"] = df["topics_from_en"].apply(lambda x: f(x, model_en, 2))
en = df.groupby("keyword").apply(lambda x: grouping(x.res_en.tolist())).reset_index(level=0)
df = pd.merge(en, vi, on="keyword")
df.columns = ["keyword", "en", "vi"]
df["en"] = df["en"].apply(lambda x: list(set(x)))
df["vi"] = df["vi"].apply(lambda x: f(x, model_vi, 3))
df["vi"] = df["vi"].apply(lambda x: list(set(x)))
df.to_csv("res_topics.csv", index=False)

df = pd.read_csv("result_web_title.csv", converters={"topics_from_en": ast.literal_eval, "topics_from_vi": ast.literal_eval})
vi = df.groupby("keyword").apply(lambda x: grouping(x.topics_from_vi.tolist())).reset_index(level=0)
df["res_en"] = df["topics_from_en"].apply(lambda x: f(x, model_en, 2))
en = df.groupby("keyword").apply(lambda x: grouping(x.res_en.tolist())).reset_index(level=0)
df = pd.merge(en, vi, on="keyword")
df.columns = ["keyword", "en", "vi"]
df["en"] = df["en"].apply(lambda x: list(set(x)))
df["vi"] = df["vi"].apply(lambda x: f(x, model_vi, 3))
df["vi"] = df["vi"].apply(lambda x: list(set(x)))
df.to_csv("res_topics_web_title.csv", index=False)