# encoding=utf-8
import jieba
import gensim
from gensim.models.doc2vec import Doc2Vec
import multiprocessing
import re
import jieba
from gensim.models.doc2vec import Doc2Vec
import multiprocessing
from bs4 import BeautifulSoup
import logging
import pymongo
import base64
from goose3 import Goose
from goose3.text import StopWordsChinese
import urllib
import time, requests
import datetime, random
import chardet
import pandas as pd
import matplotlib
import jieba as jb
import re
from collections import Counter
from wordcloud import WordCloud
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import chi2
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import f1_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
import chardet
import urllib.request
# from boilerpipe.extract import Extractor
from sklearn.decomposition import PCA
from sklearn.datasets import load_iris
from sklearn import datasets
from sklearn.cluster import KMeans
import operator
from functools import reduce
import gensim
from gensim.models.doc2vec import Doc2Vec
from sklearn import datasets
from sklearn.cluster import DBSCAN
import numpy as np  # 数据结构
import sklearn.cluster as skc  # 密度聚类
from sklearn import metrics   # 评估模型
import matplotlib.pyplot as plt  # 可视化绘图

TaggededDocument = gensim.models.doc2vec.TaggedDocument


# 定义删除除字母,数字，汉字以外的所有符号的函数
def remove_punctuation(line):
    line = str(line)
    if line.strip() == '':
        return ''
    rule = re.compile(u"[^a-zA-Z0-9\u4E00-\u9FA5]")
    line = rule.sub(' ', line)
    return line


def stopwordslist(filepath):
    stopwords = [line.strip() for line in open(filepath, 'r', encoding='utf-8').readlines()]
    return stopwords


# 加载停用词

stopwords = stopwordslist("百度停用词表.txt")


def get_datasest():
    docs_list = []
    x_train = []
    c = 0
    d = 0
    with open("test2.txt", 'r', encoding='utf-8') as cf:
        for line in cf:
            try:
                if line == '\n' or line == '\r':
                    pass
                else:
                    if len(line) > 2 :
                        each_doc = line.replace('\n', '')
                        # print(line.replace('\n', ''))
                        docs_list.append(each_doc)
                        c += 1
                        d += 1


                        for i, text in enumerate(docs_list):
                            ##如果是已经分好词的，不用再进行分词，直接按空格切分即可
                            # print(i)
                            # print(text)
                            word_list = [w for w in list(text) if w not in stopwords]
                            l = len(word_list)
                            word_list[l - 1] = word_list[l - 1].strip()
                            document = TaggededDocument(word_list, tags=[i])
                            # print(document)
                            x_train.append(document)
                        print("已经分句个数：%s ,%s" % (d, '持续训练中'))
                        docs_list = []
                        c = 0
            except Exception as e:
                print(e)

    return x_train


def train(x_train, size=100, epoch_num=1):  ##size 是你最终训练出的句子向量的维度，自己尝试着修改一下

    model_dm = Doc2Vec(x_train, min_count=1, window=10, size=size, sample=1e-3, negative=5, workers=multiprocessing.cpu_count())
    model_dm.train(x_train, total_examples=model_dm.corpus_count, epochs=70)
    model_dm.save('model_doc2vec_2019.11.27')  ##模型保存的位置
    print('语料训练完成')
    return model_dm


def ceshi(str1):
    model_dm = Doc2Vec.load("model_doc2vec_2019.11.27")
    ##此处需要读入你所需要进行提取出句子向量的文本   此处代码需要自己稍加修改一下
    ##你需要进行得到句子向量的文本，如果是分好词的，则不需要再调用结巴分词
    test_text = [w for w in list(str1)]

    inferred_vector_dm = model_dm.infer_vector(test_text)  ##得到文本的向量
    print(inferred_vector_dm)

    return inferred_vector_dm


if __name__ == '__main__':
    x_train = get_datasest()
    model_dm = train(x_train)

    # doc_2_vec = ceshi('2018年03月23日晚上大概十一点多钟我和张三骑着摩托车从住处出门想看看有什么能吃的东西.')
    # print(type(doc_2_vec))
    # print(doc_2_vec.shape)

    list_test = []
    # c = 0
    dict = {}

    with open('test2.txt', 'r', encoding='utf-8') as f:
        for line in f:
            if len(line) > 2:
                list_test.append(line.replace('\n', '').replace('\r', '').replace(' ', '').replace('   ', ''))
                print(line.replace('\n', '').replace('\r', '').replace(' ', '').replace('   ', ''))
                # dict[c] = line
                # c += 1
            else:
                pass

    TaggededDocument = gensim.models.doc2vec.TaggedDocument

    # list_line = tag.find_all('p')

    # c = 0

    list_test_indexed = []
    for i, text in enumerate(list_test):
        document = TaggededDocument(text, tags=[i])
        list_test_indexed.append(document)
        # print(document)


    # print(list_test_indexed)

    # 定义删除除字母,数字，汉字以外的所有符号的函数
    def remove_punctuation(line):
        line = str(line)
        if line.strip() == '':
            return ''
        rule = re.compile(u"[^a-zA-Z0-9\u4E00-\u9FA5]")
        line = rule.sub(' ', line)
        return line


    def stopwordslist(filepath):
        stopwords = [line.strip() for line in open(filepath, 'r', encoding='utf-8').readlines()]
        return stopwords


    # 加载停用词
    stopwords = stopwordslist("D:\Python\python_code\Liangzhi\TianPengTrans-tmp\etl\pytorch\百度停用词表.txt")


    def ceshi(str1):
        model_dm = Doc2Vec.load("model_doc2vec_2019.11.27")
        ##此处需要读入你所需要进行提取出句子向量的文本   此处代码需要自己稍加修改一下
        ##你需要进行得到句子向量的文本，如果是分好词的，则不需要再调用结巴分词
        test_text = [w for w in list(str1) if w not in stopwords]

        inferred_vector_dm = model_dm.infer_vector(test_text)  ##得到文本的向量
        # print(inferred_vector_dm)

        return inferred_vector_dm


    def myPredict(sec):
        format_sec = " ".join([w for w in list(sec) if w not in stopwords])
        ceshi(format_sec)


    matrix_all = []
    for each in list_test:
        doc_2_vec = ceshi(each)
        print(doc_2_vec)
        # print(type(doc_2_vec))
        matrix_all.append(doc_2_vec)

    output = matrix_all
    print(output)
    output_indexed = []
    for i, text in enumerate(output):
        document1 = TaggededDocument(text, tags=[i])
        output_indexed.append(document1)
        # print(document1)
    # print(output_indexed)

    X = output
    db = skc.DBSCAN(eps=0.8, min_samples=2).fit(X)  # DBSCAN聚类方法 还有参数，matric = ""距离计算方法
    labels = db.labels_  # 和X同一个维度，labels对应索引序号的值 为她所在簇的序号。若簇编号为-1，表示为噪声

    print('每个样本的簇标号:')
    print(labels)

    raito = len(labels[labels[:] == -1]) / len(labels)  # 计算噪声点个数占总数的比例
    print('噪声比:', format(raito, '.2%'))

    n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)  # 获取分簇的数目

    print('分簇的数目: %d' % n_clusters_)
    print("轮廓系数: %0.3f" % metrics.silhouette_score(X, labels))  # 轮廓系数评价聚类的好坏

    for i in range(n_clusters_):
        print('簇 ', i, '的所有样本:')
        one_cluster = np.array(X)[labels == i]
        # print(one_cluster)
        print(
            '------------------------------------------------------------------------------------------------------------')
        for each_1 in one_cluster:
            # print('参照：' + str(each_1))
            for each_index in output_indexed:
                if str(each_1) == str(each_index.words):
                    # print(each_index.words)
                    print(each_index.tags)
                    index_num = int(str(each_index.tags).replace('[', '').replace(']', ''))
                    print(list_test[index_num])
                    # plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
                    # plt.title(list_test[index_num], fontsize=10)
                    print('------------------------------------------------------------------')
        print('---------------------------------------------------------------------------------------')
        print('---------------------------------------------------------------------------------------')
        print('---------------------------------------------------------------------------------------')
        print('---------------------------------------------------------------------------------------')
f9808728d0d5a6b57a3b71faaef2dc5f4bb6f960
<mask token>


def remove_punctuation(line):
    line = str(line)
    if line.strip() == '':
        return ''
    rule = re.compile(u'[^a-zA-Z0-9一-龥]')
    line = rule.sub(' ', line)
    return line


def stopwordslist(filepath):
    stopwords = [line.strip() for line in open(filepath, 'r', encoding=
        'utf-8').readlines()]
    return stopwords


<mask token>


def get_datasest():
    docs_list = []
    x_train = []
    c = 0
    d = 0
    with open('test2.txt', 'r', encoding='utf-8') as cf:
        for line in cf:
            try:
                if line == '\n' or line == '\r':
                    pass
                elif len(line) > 2:
                    each_doc = line.replace('\n', '')
                    docs_list.append(each_doc)
                    c += 1
                    d += 1
                    for i, text in enumerate(docs_list):
                        word_list = [w for w in list(text) if w not in
                            stopwords]
                        l = len(word_list)
                        word_list[l - 1] = word_list[l - 1].strip()
                        document = TaggededDocument(word_list, tags=[i])
                        x_train.append(document)
                    print('已经分句个数：%s ,%s' % (d, '持续训练中'))
                    docs_list = []
                    c = 0
            except Exception as e:
                print(e)
    return x_train


def train(x_train, size=100, epoch_num=1):
    model_dm = Doc2Vec(x_train, min_count=1, window=10, size=size, sample=
        0.001, negative=5, workers=multiprocessing.cpu_count())
    model_dm.train(x_train, total_examples=model_dm.corpus_count, epochs=70)
    model_dm.save('model_doc2vec_2019.11.27')
    print('语料训练完成')
    return model_dm


def ceshi(str1):
    model_dm = Doc2Vec.load('model_doc2vec_2019.11.27')
    test_text = [w for w in list(str1)]
    inferred_vector_dm = model_dm.infer_vector(test_text)
    print(inferred_vector_dm)
    return inferred_vector_dm


<mask token>

<mask token>


def remove_punctuation(line):
    line = str(line)
    if line.strip() == '':
        return ''
    rule = re.compile(u'[^a-zA-Z0-9一-龥]')
    line = rule.sub(' ', line)
    return line


def stopwordslist(filepath):
    stopwords = [line.strip() for line in open(filepath, 'r', encoding=
        'utf-8').readlines()]
    return stopwords


<mask token>


def get_datasest():
    docs_list = []
    x_train = []
    c = 0
    d = 0
    with open('test2.txt', 'r', encoding='utf-8') as cf:
        for line in cf:
            try:
                if line == '\n' or line == '\r':
                    pass
                elif len(line) > 2:
                    each_doc = line.replace('\n', '')
                    docs_list.append(each_doc)
                    c += 1
                    d += 1
                    for i, text in enumerate(docs_list):
                        word_list = [w for w in list(text) if w not in
                            stopwords]
                        l = len(word_list)
                        word_list[l - 1] = word_list[l - 1].strip()
                        document = TaggededDocument(word_list, tags=[i])
                        x_train.append(document)
                    print('已经分句个数：%s ,%s' % (d, '持续训练中'))
                    docs_list = []
                    c = 0
            except Exception as e:
                print(e)
    return x_train


def train(x_train, size=100, epoch_num=1):
    model_dm = Doc2Vec(x_train, min_count=1, window=10, size=size, sample=
        0.001, negative=5, workers=multiprocessing.cpu_count())
    model_dm.train(x_train, total_examples=model_dm.corpus_count, epochs=70)
    model_dm.save('model_doc2vec_2019.11.27')
    print('语料训练完成')
    return model_dm


def ceshi(str1):
    model_dm = Doc2Vec.load('model_doc2vec_2019.11.27')
    test_text = [w for w in list(str1)]
    inferred_vector_dm = model_dm.infer_vector(test_text)
    print(inferred_vector_dm)
    return inferred_vector_dm


if __name__ == '__main__':
    x_train = get_datasest()
    model_dm = train(x_train)
    list_test = []
    dict = {}
    with open('test2.txt', 'r', encoding='utf-8') as f:
        for line in f:
            if len(line) > 2:
                list_test.append(line.replace('\n', '').replace('\r', '').
                    replace(' ', '').replace('   ', ''))
                print(line.replace('\n', '').replace('\r', '').replace(' ',
                    '').replace('   ', ''))
            else:
                pass
    TaggededDocument = gensim.models.doc2vec.TaggedDocument
    list_test_indexed = []
    for i, text in enumerate(list_test):
        document = TaggededDocument(text, tags=[i])
        list_test_indexed.append(document)

    def remove_punctuation(line):
        line = str(line)
        if line.strip() == '':
            return ''
        rule = re.compile(u'[^a-zA-Z0-9一-龥]')
        line = rule.sub(' ', line)
        return line

    def stopwordslist(filepath):
        stopwords = [line.strip() for line in open(filepath, 'r', encoding=
            'utf-8').readlines()]
        return stopwords
    stopwords = stopwordslist(
        'D:\\Python\\python_code\\Liangzhi\\TianPengTrans-tmp\\etl\\pytorch\\百度停用词表.txt'
        )

    def ceshi(str1):
        model_dm = Doc2Vec.load('model_doc2vec_2019.11.27')
        test_text = [w for w in list(str1) if w not in stopwords]
        inferred_vector_dm = model_dm.infer_vector(test_text)
        return inferred_vector_dm

    def myPredict(sec):
        format_sec = ' '.join([w for w in list(sec) if w not in stopwords])
        ceshi(format_sec)
    matrix_all = []
    for each in list_test:
        doc_2_vec = ceshi(each)
        print(doc_2_vec)
        matrix_all.append(doc_2_vec)
    output = matrix_all
    print(output)
    output_indexed = []
    for i, text in enumerate(output):
        document1 = TaggededDocument(text, tags=[i])
        output_indexed.append(document1)
    X = output
    db = skc.DBSCAN(eps=0.8, min_samples=2).fit(X)
    labels = db.labels_
    print('每个样本的簇标号:')
    print(labels)
    raito = len(labels[labels[:] == -1]) / len(labels)
    print('噪声比:', format(raito, '.2%'))
    n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
    print('分簇的数目: %d' % n_clusters_)
    print('轮廓系数: %0.3f' % metrics.silhouette_score(X, labels))
    for i in range(n_clusters_):
        print('簇 ', i, '的所有样本:')
        one_cluster = np.array(X)[labels == i]
        print(
            '------------------------------------------------------------------------------------------------------------'
            )
        for each_1 in one_cluster:
            for each_index in output_indexed:
                if str(each_1) == str(each_index.words):
                    print(each_index.tags)
                    index_num = int(str(each_index.tags).replace('[', '').
                        replace(']', ''))
                    print(list_test[index_num])
                    print(
                        '------------------------------------------------------------------'
                        )
        print(
            '---------------------------------------------------------------------------------------'
            )
        print(
            '---------------------------------------------------------------------------------------'
            )
        print(
            '---------------------------------------------------------------------------------------'
            )
        print(
            '---------------------------------------------------------------------------------------'
            )

<mask token>
TaggededDocument = gensim.models.doc2vec.TaggedDocument


def remove_punctuation(line):
    line = str(line)
    if line.strip() == '':
        return ''
    rule = re.compile(u'[^a-zA-Z0-9一-龥]')
    line = rule.sub(' ', line)
    return line


def stopwordslist(filepath):
    stopwords = [line.strip() for line in open(filepath, 'r', encoding=
        'utf-8').readlines()]
    return stopwords


stopwords = stopwordslist('百度停用词表.txt')


def get_datasest():
    docs_list = []
    x_train = []
    c = 0
    d = 0
    with open('test2.txt', 'r', encoding='utf-8') as cf:
        for line in cf:
            try:
                if line == '\n' or line == '\r':
                    pass
                elif len(line) > 2:
                    each_doc = line.replace('\n', '')
                    docs_list.append(each_doc)
                    c += 1
                    d += 1
                    for i, text in enumerate(docs_list):
                        word_list = [w for w in list(text) if w not in
                            stopwords]
                        l = len(word_list)
                        word_list[l - 1] = word_list[l - 1].strip()
                        document = TaggededDocument(word_list, tags=[i])
                        x_train.append(document)
                    print('已经分句个数：%s ,%s' % (d, '持续训练中'))
                    docs_list = []
                    c = 0
            except Exception as e:
                print(e)
    return x_train


def train(x_train, size=100, epoch_num=1):
    model_dm = Doc2Vec(x_train, min_count=1, window=10, size=size, sample=
        0.001, negative=5, workers=multiprocessing.cpu_count())
    model_dm.train(x_train, total_examples=model_dm.corpus_count, epochs=70)
    model_dm.save('model_doc2vec_2019.11.27')
    print('语料训练完成')
    return model_dm


def ceshi(str1):
    model_dm = Doc2Vec.load('model_doc2vec_2019.11.27')
    test_text = [w for w in list(str1)]
    inferred_vector_dm = model_dm.infer_vector(test_text)
    print(inferred_vector_dm)
    return inferred_vector_dm


if __name__ == '__main__':
    x_train = get_datasest()
    model_dm = train(x_train)
    list_test = []
    dict = {}
    with open('test2.txt', 'r', encoding='utf-8') as f:
        for line in f:
            if len(line) > 2:
                list_test.append(line.replace('\n', '').replace('\r', '').
                    replace(' ', '').replace('   ', ''))
                print(line.replace('\n', '').replace('\r', '').replace(' ',
                    '').replace('   ', ''))
            else:
                pass
    TaggededDocument = gensim.models.doc2vec.TaggedDocument
    list_test_indexed = []
    for i, text in enumerate(list_test):
        document = TaggededDocument(text, tags=[i])
        list_test_indexed.append(document)

    def remove_punctuation(line):
        line = str(line)
        if line.strip() == '':
            return ''
        rule = re.compile(u'[^a-zA-Z0-9一-龥]')
        line = rule.sub(' ', line)
        return line

    def stopwordslist(filepath):
        stopwords = [line.strip() for line in open(filepath, 'r', encoding=
            'utf-8').readlines()]
        return stopwords
    stopwords = stopwordslist(
        'D:\\Python\\python_code\\Liangzhi\\TianPengTrans-tmp\\etl\\pytorch\\百度停用词表.txt'
        )

    def ceshi(str1):
        model_dm = Doc2Vec.load('model_doc2vec_2019.11.27')
        test_text = [w for w in list(str1) if w not in stopwords]
        inferred_vector_dm = model_dm.infer_vector(test_text)
        return inferred_vector_dm

    def myPredict(sec):
        format_sec = ' '.join([w for w in list(sec) if w not in stopwords])
        ceshi(format_sec)
    matrix_all = []
    for each in list_test:
        doc_2_vec = ceshi(each)
        print(doc_2_vec)
        matrix_all.append(doc_2_vec)
    output = matrix_all
    print(output)
    output_indexed = []
    for i, text in enumerate(output):
        document1 = TaggededDocument(text, tags=[i])
        output_indexed.append(document1)
    X = output
    db = skc.DBSCAN(eps=0.8, min_samples=2).fit(X)
    labels = db.labels_
    print('每个样本的簇标号:')
    print(labels)
    raito = len(labels[labels[:] == -1]) / len(labels)
    print('噪声比:', format(raito, '.2%'))
    n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
    print('分簇的数目: %d' % n_clusters_)
    print('轮廓系数: %0.3f' % metrics.silhouette_score(X, labels))
    for i in range(n_clusters_):
        print('簇 ', i, '的所有样本:')
        one_cluster = np.array(X)[labels == i]
        print(
            '------------------------------------------------------------------------------------------------------------'
            )
        for each_1 in one_cluster:
            for each_index in output_indexed:
                if str(each_1) == str(each_index.words):
                    print(each_index.tags)
                    index_num = int(str(each_index.tags).replace('[', '').
                        replace(']', ''))
                    print(list_test[index_num])
                    print(
                        '------------------------------------------------------------------'
                        )
        print(
            '---------------------------------------------------------------------------------------'
            )
        print(
            '---------------------------------------------------------------------------------------'
            )
        print(
            '---------------------------------------------------------------------------------------'
            )
        print(
            '---------------------------------------------------------------------------------------'
            )

import jieba
import gensim
from gensim.models.doc2vec import Doc2Vec
import multiprocessing
import re
import jieba
from gensim.models.doc2vec import Doc2Vec
import multiprocessing
from bs4 import BeautifulSoup
import logging
import pymongo
import base64
from goose3 import Goose
from goose3.text import StopWordsChinese
import urllib
import time, requests
import datetime, random
import chardet
import pandas as pd
import matplotlib
import jieba as jb
import re
from collections import Counter
from wordcloud import WordCloud
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import chi2
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import f1_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
import chardet
import urllib.request
from sklearn.decomposition import PCA
from sklearn.datasets import load_iris
from sklearn import datasets
from sklearn.cluster import KMeans
import operator
from functools import reduce
import gensim
from gensim.models.doc2vec import Doc2Vec
from sklearn import datasets
from sklearn.cluster import DBSCAN
import numpy as np
import sklearn.cluster as skc
from sklearn import metrics
import matplotlib.pyplot as plt
TaggededDocument = gensim.models.doc2vec.TaggedDocument


def remove_punctuation(line):
    line = str(line)
    if line.strip() == '':
        return ''
    rule = re.compile(u'[^a-zA-Z0-9一-龥]')
    line = rule.sub(' ', line)
    return line


def stopwordslist(filepath):
    stopwords = [line.strip() for line in open(filepath, 'r', encoding=
        'utf-8').readlines()]
    return stopwords


stopwords = stopwordslist('百度停用词表.txt')


def get_datasest():
    docs_list = []
    x_train = []
    c = 0
    d = 0
    with open('test2.txt', 'r', encoding='utf-8') as cf:
        for line in cf:
            try:
                if line == '\n' or line == '\r':
                    pass
                elif len(line) > 2:
                    each_doc = line.replace('\n', '')
                    docs_list.append(each_doc)
                    c += 1
                    d += 1
                    for i, text in enumerate(docs_list):
                        word_list = [w for w in list(text) if w not in
                            stopwords]
                        l = len(word_list)
                        word_list[l - 1] = word_list[l - 1].strip()
                        document = TaggededDocument(word_list, tags=[i])
                        x_train.append(document)
                    print('已经分句个数：%s ,%s' % (d, '持续训练中'))
                    docs_list = []
                    c = 0
            except Exception as e:
                print(e)
    return x_train


def train(x_train, size=100, epoch_num=1):
    model_dm = Doc2Vec(x_train, min_count=1, window=10, size=size, sample=
        0.001, negative=5, workers=multiprocessing.cpu_count())
    model_dm.train(x_train, total_examples=model_dm.corpus_count, epochs=70)
    model_dm.save('model_doc2vec_2019.11.27')
    print('语料训练完成')
    return model_dm


def ceshi(str1):
    model_dm = Doc2Vec.load('model_doc2vec_2019.11.27')
    test_text = [w for w in list(str1)]
    inferred_vector_dm = model_dm.infer_vector(test_text)
    print(inferred_vector_dm)
    return inferred_vector_dm


if __name__ == '__main__':
    x_train = get_datasest()
    model_dm = train(x_train)
    list_test = []
    dict = {}
    with open('test2.txt', 'r', encoding='utf-8') as f:
        for line in f:
            if len(line) > 2:
                list_test.append(line.replace('\n', '').replace('\r', '').
                    replace(' ', '').replace('   ', ''))
                print(line.replace('\n', '').replace('\r', '').replace(' ',
                    '').replace('   ', ''))
            else:
                pass
    TaggededDocument = gensim.models.doc2vec.TaggedDocument
    list_test_indexed = []
    for i, text in enumerate(list_test):
        document = TaggededDocument(text, tags=[i])
        list_test_indexed.append(document)

    def remove_punctuation(line):
        line = str(line)
        if line.strip() == '':
            return ''
        rule = re.compile(u'[^a-zA-Z0-9一-龥]')
        line = rule.sub(' ', line)
        return line

    def stopwordslist(filepath):
        stopwords = [line.strip() for line in open(filepath, 'r', encoding=
            'utf-8').readlines()]
        return stopwords
    stopwords = stopwordslist(
        'D:\\Python\\python_code\\Liangzhi\\TianPengTrans-tmp\\etl\\pytorch\\百度停用词表.txt'
        )

    def ceshi(str1):
        model_dm = Doc2Vec.load('model_doc2vec_2019.11.27')
        test_text = [w for w in list(str1) if w not in stopwords]
        inferred_vector_dm = model_dm.infer_vector(test_text)
        return inferred_vector_dm

    def myPredict(sec):
        format_sec = ' '.join([w for w in list(sec) if w not in stopwords])
        ceshi(format_sec)
    matrix_all = []
    for each in list_test:
        doc_2_vec = ceshi(each)
        print(doc_2_vec)
        matrix_all.append(doc_2_vec)
    output = matrix_all
    print(output)
    output_indexed = []
    for i, text in enumerate(output):
        document1 = TaggededDocument(text, tags=[i])
        output_indexed.append(document1)
    X = output
    db = skc.DBSCAN(eps=0.8, min_samples=2).fit(X)
    labels = db.labels_
    print('每个样本的簇标号:')
    print(labels)
    raito = len(labels[labels[:] == -1]) / len(labels)
    print('噪声比:', format(raito, '.2%'))
    n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
    print('分簇的数目: %d' % n_clusters_)
    print('轮廓系数: %0.3f' % metrics.silhouette_score(X, labels))
    for i in range(n_clusters_):
        print('簇 ', i, '的所有样本:')
        one_cluster = np.array(X)[labels == i]
        print(
            '------------------------------------------------------------------------------------------------------------'
            )
        for each_1 in one_cluster:
            for each_index in output_indexed:
                if str(each_1) == str(each_index.words):
                    print(each_index.tags)
                    index_num = int(str(each_index.tags).replace('[', '').
                        replace(']', ''))
                    print(list_test[index_num])
                    print(
                        '------------------------------------------------------------------'
                        )
        print(
            '---------------------------------------------------------------------------------------'
            )
        print(
            '---------------------------------------------------------------------------------------'
            )
        print(
            '---------------------------------------------------------------------------------------'
            )
        print(
            '---------------------------------------------------------------------------------------'
            )

# encoding=utf-8
import jieba
import gensim
from gensim.models.doc2vec import Doc2Vec
import multiprocessing
import re
import jieba
from gensim.models.doc2vec import Doc2Vec
import multiprocessing
from bs4 import BeautifulSoup
import logging
import pymongo
import base64
from goose3 import Goose
from goose3.text import StopWordsChinese
import urllib
import time, requests
import datetime, random
import chardet
import pandas as pd
import matplotlib
import jieba as jb
import re
from collections import Counter
from wordcloud import WordCloud
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import chi2
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import f1_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
import chardet
import urllib.request
# from boilerpipe.extract import Extractor
from sklearn.decomposition import PCA
from sklearn.datasets import load_iris
from sklearn import datasets
from sklearn.cluster import KMeans
import operator
from functools import reduce
import gensim
from gensim.models.doc2vec import Doc2Vec
from sklearn import datasets
from sklearn.cluster import DBSCAN
import numpy as np  # 数据结构
import sklearn.cluster as skc  # 密度聚类
from sklearn import metrics   # 评估模型
import matplotlib.pyplot as plt  # 可视化绘图

TaggededDocument = gensim.models.doc2vec.TaggedDocument


# 定义删除除字母,数字，汉字以外的所有符号的函数
def remove_punctuation(line):
    line = str(line)
    if line.strip() == '':
        return ''
    rule = re.compile(u"[^a-zA-Z0-9\u4E00-\u9FA5]")
    line = rule.sub(' ', line)
    return line


def stopwordslist(filepath):
    stopwords = [line.strip() for line in open(filepath, 'r', encoding='utf-8').readlines()]
    return stopwords


# 加载停用词

stopwords = stopwordslist("百度停用词表.txt")


def get_datasest():
    docs_list = []
    x_train = []
    c = 0
    d = 0
    with open("test2.txt", 'r', encoding='utf-8') as cf:
        for line in cf:
            try:
                if line == '\n' or line == '\r':
                    pass
                else:
                    if len(line) > 2 :
                        each_doc = line.replace('\n', '')
                        # print(line.replace('\n', ''))
                        docs_list.append(each_doc)
                        c += 1
                        d += 1


                        for i, text in enumerate(docs_list):
                            ##如果是已经分好词的，不用再进行分词，直接按空格切分即可
                            # print(i)
                            # print(text)
                            word_list = [w for w in list(text) if w not in stopwords]
                            l = len(word_list)
                            word_list[l - 1] = word_list[l - 1].strip()
                            document = TaggededDocument(word_list, tags=[i])
                            # print(document)
                            x_train.append(document)
                        print("已经分句个数：%s ,%s" % (d, '持续训练中'))
                        docs_list = []
                        c = 0
            except Exception as e:
                print(e)

    return x_train


def train(x_train, size=100, epoch_num=1):  ##size 是你最终训练出的句子向量的维度，自己尝试着修改一下

    model_dm = Doc2Vec(x_train, min_count=1, window=10, size=size, sample=1e-3, negative=5, workers=multiprocessing.cpu_count())
    model_dm.train(x_train, total_examples=model_dm.corpus_count, epochs=70)
    model_dm.save('model_doc2vec_2019.11.27')  ##模型保存的位置
    print('语料训练完成')
    return model_dm


def ceshi(str1):
    model_dm = Doc2Vec.load("model_doc2vec_2019.11.27")
    ##此处需要读入你所需要进行提取出句子向量的文本   此处代码需要自己稍加修改一下
    ##你需要进行得到句子向量的文本，如果是分好词的，则不需要再调用结巴分词
    test_text = [w for w in list(str1)]

    inferred_vector_dm = model_dm.infer_vector(test_text)  ##得到文本的向量
    print(inferred_vector_dm)

    return inferred_vector_dm


if __name__ == '__main__':
    x_train = get_datasest()
    model_dm = train(x_train)

    # doc_2_vec = ceshi('2018年03月23日晚上大概十一点多钟我和张三骑着摩托车从住处出门想看看有什么能吃的东西.')
    # print(type(doc_2_vec))
    # print(doc_2_vec.shape)

    list_test = []
    # c = 0
    dict = {}

    with open('test2.txt', 'r', encoding='utf-8') as f:
        for line in f:
            if len(line) > 2:
                list_test.append(line.replace('\n', '').replace('\r', '').replace(' ', '').replace('   ', ''))
                print(line.replace('\n', '').replace('\r', '').replace(' ', '').replace('   ', ''))
                # dict[c] = line
                # c += 1
            else:
                pass

    TaggededDocument = gensim.models.doc2vec.TaggedDocument

    # list_line = tag.find_all('p')

    # c = 0

    list_test_indexed = []
    for i, text in enumerate(list_test):
        document = TaggededDocument(text, tags=[i])
        list_test_indexed.append(document)
        # print(document)


    # print(list_test_indexed)

    # 定义删除除字母,数字，汉字以外的所有符号的函数
    def remove_punctuation(line):
        line = str(line)
        if line.strip() == '':
            return ''
        rule = re.compile(u"[^a-zA-Z0-9\u4E00-\u9FA5]")
        line = rule.sub(' ', line)
        return line


    def stopwordslist(filepath):
        stopwords = [line.strip() for line in open(filepath, 'r', encoding='utf-8').readlines()]
        return stopwords


    # 加载停用词
    stopwords = stopwordslist("D:\Python\python_code\Liangzhi\TianPengTrans-tmp\etl\pytorch\百度停用词表.txt")


    def ceshi(str1):
        model_dm = Doc2Vec.load("model_doc2vec_2019.11.27")
        ##此处需要读入你所需要进行提取出句子向量的文本   此处代码需要自己稍加修改一下
        ##你需要进行得到句子向量的文本，如果是分好词的，则不需要再调用结巴分词
        test_text = [w for w in list(str1) if w not in stopwords]

        inferred_vector_dm = model_dm.infer_vector(test_text)  ##得到文本的向量
        # print(inferred_vector_dm)

        return inferred_vector_dm


    def myPredict(sec):
        format_sec = " ".join([w for w in list(sec) if w not in stopwords])
        ceshi(format_sec)


    matrix_all = []
    for each in list_test:
        doc_2_vec = ceshi(each)
        print(doc_2_vec)
        # print(type(doc_2_vec))
        matrix_all.append(doc_2_vec)

    output = matrix_all
    print(output)
    output_indexed = []
    for i, text in enumerate(output):
        document1 = TaggededDocument(text, tags=[i])
        output_indexed.append(document1)
        # print(document1)
    # print(output_indexed)

    X = output
    db = skc.DBSCAN(eps=0.8, min_samples=2).fit(X)  # DBSCAN聚类方法 还有参数，matric = ""距离计算方法
    labels = db.labels_  # 和X同一个维度，labels对应索引序号的值 为她所在簇的序号。若簇编号为-1，表示为噪声

    print('每个样本的簇标号:')
    print(labels)

    raito = len(labels[labels[:] == -1]) / len(labels)  # 计算噪声点个数占总数的比例
    print('噪声比:', format(raito, '.2%'))

    n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)  # 获取分簇的数目

    print('分簇的数目: %d' % n_clusters_)
    print("轮廓系数: %0.3f" % metrics.silhouette_score(X, labels))  # 轮廓系数评价聚类的好坏

    for i in range(n_clusters_):
        print('簇 ', i, '的所有样本:')
        one_cluster = np.array(X)[labels == i]
        # print(one_cluster)
        print(
            '------------------------------------------------------------------------------------------------------------')
        for each_1 in one_cluster:
            # print('参照：' + str(each_1))
            for each_index in output_indexed:
                if str(each_1) == str(each_index.words):
                    # print(each_index.words)
                    print(each_index.tags)
                    index_num = int(str(each_index.tags).replace('[', '').replace(']', ''))
                    print(list_test[index_num])
                    # plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
                    # plt.title(list_test[index_num], fontsize=10)
                    print('------------------------------------------------------------------')
        print('---------------------------------------------------------------------------------------')
        print('---------------------------------------------------------------------------------------')
        print('---------------------------------------------------------------------------------------')
        print('---------------------------------------------------------------------------------------')