# -*- coding: utf-8 -*-
"""
Created on Sat Feb 24 14:19:00 2018

@author: fangyucheng
"""

import os
import time
import aiohttp
import asyncio
import requests
from bs4 import BeautifulSoup
import datetime
import re
import json
import copy
import random
# from selenium import webdriver
from multiprocessing import Pool
from multiprocessing import Process
from crawler_sys.framework.video_fields_std import Std_fields_video
from crawler_sys.utils.output_results import retry_get_url
from crawler_sys.utils.output_results import output_result
from crawler_sys.utils import trans_strtime_to_timestamp
from crawler.crawler_sys.utils import connect_with_redis
from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
from crawler.crawler_sys.utils.util_logging import logged


class Crawler_youku():

    def __init__(self, timeout=None, platform='youku'):
        if timeout==None:
            self.timeout = 10
        else:
            self.timeout = timeout
        self.platform = platform
        std_fields = Std_fields_video()
        self.video_data = std_fields.video_data
        self.video_data['platform'] = self.platform
        # remove fields that crawled data don't have
        pop_key_Lst = ['channel', 'describe', 'repost_count', 'isOriginal',
                       'video_id']
        for popk in pop_key_Lst:
            self.video_data.pop(popk)

        # with channel == '全部', url field is hard coded in list_page_all method,
        # input url will be ignored
        self.legal_channels = ['全部']
        self.list_page_url_dict = {
            '全部': '_',
            }

    def video_page(self, url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('span',{'id': "module_basic_sub"}).a.text
        releaser = releaser.replace('\n', '')
        releaser = releaser.replace(' ', '')
        releaserUrl = soup.find('span',{'id': "module_basic_sub"}).a['href']
        if releaserUrl[0:4] == 'http':
            pass
        else:
            releaserUrl = 'http:' + releaserUrl
        try:
            rt_str = soup.find('span', {'class':'video-status'}).text
            rt_lst = re.findall('\d+', rt_str)
            rt_to_timestamp = str(rt_lst[0]) + '-' + str(rt_lst[1]) + '-' + str(rt_lst[2])
            release_time = int(datetime.datetime.strptime(rt_to_timestamp, '%Y-%m-%d').timestamp()*1e3)
        except:
            release_time = 0
        D0 = {'releaser':releaser,'releaserUrl':releaserUrl,'release_time':release_time}
        return D0

    def get_video_infor_by_network(self,url_Lst):
        video_id_Lst = []
        for url in url_Lst:
            video_id = ' '.join(re.findall('id.*html',url)).split('_')[1].split('.')[0]
            video_id_Lst.append(video_id)


    def get_video_title_play_count_duration(self, url):
        video_id = ' '.join(re.findall('id.*html', url))
        mobile_url = 'https://m.youku.com/video/' + video_id
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        title = browser.find_element_by_class_name('online').text
        dura = browser.find_element_by_class_name('v-label').text.split('\n')[1]
        dura_lst = ' '.join(dura).split[':']
        if len(dura_lst) == 2:
            duration = dura_lst[0]*60 + dura_lst[1]
        elif len(dura_lst) == 3:
            duration = dura_lst[0]*3600 + dura_lst[1]*60 + dura_lst[2]
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',','')
        pc_midstep4 = re.findall('[0-9]+',pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0]*1e4+pc_midstep4[1]*1e3
        else:
            play_count = ' '.join(pc_midstep4)
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
        D0 = {'video_id': video_id,
              'title': title,
              'duration': duration,
              'play_count': play_count,
              'url': url,
              'fetch_time': fetch_time}
        browser.close()
        return D0

    def get_video_play_count_by_vid(self, vid):
        mobile_url = 'https://m.youku.com/video/id_' + vid + '==.html'
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        time.sleep(10)
        title = browser.find_element_by_class_name('online').text
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',','')
        pc_midstep4 = re.findall('[0-9]+', pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0]*1e4+pc_midstep4[1]*1e3
        else:
            play_count = ' '.join(pc_midstep4)
        browser.close()
        D0 = {'title': title,
              'play_count': play_count}
        return D0

    def get_video_title_releaser_release_time(self, url):
        """An example of selenium
        """
        video_id = ' '.join(re.findall('id.*html', url))
        browser = webdriver.Chrome()
        browser.get(url)
        title = browser.find_element_by_id('subtitle').text
        releaser = browser.find_element_by_id('module_basic_sub').text
        releaser = releaser.replace('+订阅','')
        releaser = releaser.replace(' ','')
        try:
            rt_midstep = browser.find_element_by_class_name('video-status').text
            rt_midstep = rt_midstep.replace('上传于','')
            rt_midstep = rt_midstep.replace(' ','')
            release_time = int(datetime.datetime.strptime(rt_midstep,'%Y-%m-%d').timestamp()*1e3)
        except:
            release_time = 0
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
        D0 = {'video_id': video_id,
              'title': title,
              'release_time': release_time,
              'url': url,
              'fetch_time': fetch_time}
        return D0

    def list_page_sync(self, list_page_url, channel=None,
                  output_to_file=False, filepath=None,
                  output_to_es_raw=False,
                  output_to_es_register=False,
                  push_to_redis=False,
                  page_num_max=30):
        if channel == '全部':
            self.list_page_all(output_to_file=output_to_file, filepath=filepath,
                               output_to_es_raw=output_to_es_raw,
                               output_to_es_register=output_to_es_register,
                               push_to_redis=push_to_redis,
                               page_num_max=page_num_max)
        else:
            # need to be tested
            get_page = requests.get(list_page_url)
            page = get_page.text
            soup = BeautifulSoup(page,'html.parser')
            video = soup.find_all('div',{'class':'v-link'})
            dict_url = []
            for ttt in video:
                midstepurl=ttt.find('a')['href']
                if ('https:' in midstepurl):
                    midstepurl=midstepurl[6:]
                url = 'http:'+midstepurl
                dict_url.append(url)
            for xxxx in dict_url:
                if ("id" not in xxxx):
                    num=dict_url.index(xxxx)
                    dict_url.pop(num)
            for xxxx in dict_url:
                if ("id" not in xxxx):
                    num=dict_url.index(xxxx)
                    dict_url.pop(num)
            list_page_Lst = []
            for videourl in dict_url:
                one_video_dic = self.video_page(videourl)
                print ('one page done')
                list_page_Lst.append(one_video_dic)
            return list_page_Lst


    def search_page(self, keyword, search_pages_max=10,
                    output_to_es_raw=False,
                    output_to_es_register=False,
                    es_index=None,
                    doc_type=None):
        def process_one_line(data_line):
            title = data_line.find("div", {"class" : "v-meta-title"}).a['title']
            url = data_line.find("div", {"class" : "v-meta-title"}).a['href']
            releaser = data_line.find("span", {"class" : "username"}).text
            releaser = releaser.strip()
            releaserUrl = data_line.find("span", {"class" : "username"}).a['href']
            releaserUrl = 'http:' + releaserUrl
            play_count = data_line.find("span", {"class" : "pub"}).text
            dura = data_line.find("span", {"class" : "v-time"}).text
            duration_str = dura
            dl = duration_str.split(':')
            dl_int = []
            for v in dl:
                v = int(v)
                dl_int.append(v)
            if len(dl_int) == 2:
                duration = dl_int[0]*60+dl_int[1]
            else:
                duration = dl_int[0]*3660 + dl_int[1]*60+dl_int[2]
            videourl = 'https:' + url
            dict_page = self.video_page(videourl)
            release_time = dict_page['release_time']
            D0 = copy.deepcopy(self.video_data)
            D0['title'] = title
            D0['url'] = videourl
            D0['duration'] = duration
            D0['releaser'] = releaser
            D0['release_time'] = release_time
            D0['releaserUrl'] = releaserUrl
            D0['play_count'] = play_count

            return D0

        search_data_Lst = []
        jsurl = ('http://www.soku.com/search_video_ajax/q_'
                 + keyword
                 + '_orderby_1_limitdate_0?site=14&page={}'.format(
                     str(i)) for i in range (2, 4))
        for urls in jsurl:
            print(urls)
            get_page = requests.get(urls)
            print('get jspage done')
            page = get_page.text
            soup = BeautifulSoup(page,'html.parser')
            youku = soup.find_all("div", { "class" : "v" })
            for data_line in youku:
                one_video_dic = process_one_line(data_line)
                search_data_Lst.append(one_video_dic)
        url=('http://www.soku.com/search_video/q_'
             + keyword
             + '_orderby_1_limitdate_0?spm=a2h0k.8191407.0.0&site=14&page={}'.format(
                 str(i)) for i in range(1, search_pages_max+1))
        for urls in url:
            print(urls)
            get_page = requests.get(urls)
            print('get page done')
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            youku = soup.find_all("div", {"class" : "v"})
            for data_line in youku:
                one_video_dic = process_one_line(data_line)
                search_data_Lst.append(one_video_dic)
                if len(search_data_Lst) >= 100:
                    output_result(result_Lst=search_data_Lst,
                                  platform=self.platform,
                                  output_to_es_raw=output_to_es_raw,
                                  output_to_es_register=output_to_es_register,
                                  es_index=es_index,
                                  doc_type=doc_type)
                    search_data_Lst.clear()

        if search_data_Lst != []:
            output_result(result_Lst=search_data_Lst,
                          platform=self.platform,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          es_index=es_index,
                          doc_type=doc_type)

        return search_data_Lst


    def get_releaser_detail(self, url):
        releaser_data_Lst = []
        get_page = requests.get(url)
        page = get_page.text
        soup = BeautifulSoup(page,'html.parser')
        releaser=soup.find('a',{'class':'username'})['title']
        releaser_intro=soup.find('span',{'class':'desc'}).text
        total_play_count=soup.find('li',{'class':'vnum'})['title']
        fans=soup.find('li',{'class':'snum'})['title']
        midstep=soup.find_all('div',{'class':'title'})
        for brother in midstep:
            if brother.h3.text=='视频':
                num=brother.find('span').text
                self.TotalVideo_num=num[1:-1]
        D_releaser_page={"releaser":releaser,"releaser_intro":releaser_intro,"total_play_count":total_play_count,"follow":fans,"TotalVideo_num":self.TotalVideo_num}
        releaser_data_Lst.append(D_releaser_page)
        return D_releaser_page


    def get_releaser_info(self, releaserUrl):
        get_page = retry_get_url(releaserUrl)
        page = get_page.text
        soup = BeautifulSoup(page,'html.parser')
        releaser = soup.find('a',{'class':'username'})['title']
        releaser_id = soup.find('a',{'class':'user-avatar'})['href']
        #total_video_num = soup.find('div',{'class':'title'}).text
        D0 = {'releaser':releaser, 'releaser_id':releaser_id}
        return D0

    @logged
    def releaser_page(self, releaserUrl, error_file=None,
                      output_to_file=False, filepath=None,
                      output_to_es_raw=False,
                      output_es_index=None,
                      output_doc_type=None,
                      output_to_es_register=False,
                      push_to_redis=False,
                      releaser_page_num_max=300):
        result_Lst = []
        releaser_info = self.get_releaser_info(releaserUrl)
        releaser = releaser_info['releaser']
        releaser_id = releaser_info['releaser_id']
        count = 1
        while count <= releaser_page_num_max:
            releaser_url = 'https://i.youku.com' + releaser_id + '/videos?&order=1&page=' + str(count)
            count += 1
            get_page = retry_get_url(releaser_url)
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_lst = soup.find_all("div", { "class" : "v va" })
            for one_video in video_lst:
                try:
                    title = one_video.find("div",{"class":"v-meta-title"}).a['title']
                    url = one_video.find('div',{'class':'v-meta-title'}).a['href']
                    if url[0:4] != 'http':
                        url = 'https:' + url
                    try:
                        play_count_text = one_video.find("span",{"class":"v-num"}).text
                        if '万' in play_count_text:
                            play_count_str = play_count_text.replace('万', '')
                            play_count = int(float(play_count_str)*1e4)
                        else:
                            play_count = int(play_count_text.replace(',', ''))
                    except:
                        play_count = 0
                    dura_str = one_video.find("span",{"class":"v-time"}).text
                    dura_lst = dura_str.split(':')
                    if len(dura_lst) == 2:
                        duration = int(dura_lst[0])*60 + int(dura_lst[1])
                    elif len(dura_lst) == 3:
                        duration = int(dura_lst[0])*3600 + int(dura_lst[1])*60 + int(dura_lst[2])
                    else:
                        duration = 0
                    rt_str = one_video.find('span',{'class':'v-publishtime'}).text
                    release_time = trans_strtime_to_timestamp.trans_strtime_to_timestamp(rt_str, missing_year=True)
                    fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)

                    self.video_data['title'] = title
                    self.video_data['duration'] = duration
                    self.video_data['url'] = url
                    self.video_data['play_count'] = play_count
                    self.video_data['releaser'] = releaser
                    self.video_data['releaserUrl'] = releaserUrl
                    self.video_data['release_time'] = release_time
                    self.video_data['fetch_time'] = fetch_time

                    get_data = copy.deepcopy(self.video_data)
                    result_Lst.append(get_data)
                    print('get_one_line %s' % url)
                except:
                    print('failed to get data')
                    pass
                if len(result_Lst)%100 == 0:
                    if output_doc_type is None and output_es_index is None:
                        output_result(result_Lst,
                                      self.platform,
                                      output_to_file=output_to_file,
                                      filepath=filepath,
                                      output_to_es_raw=output_to_es_raw,
                                      output_to_es_register=output_to_es_register,
                                      push_to_redis=push_to_redis)
                        result_Lst.clear()

                    elif output_doc_type is not None and output_es_index is not None:
                        output_result(result_Lst,
                                      self.platform,
                                      output_to_file=output_to_file,
                                      filepath=filepath,
                                      output_to_es_raw=output_to_es_raw,
                                      output_to_es_register=output_to_es_register,
                                      push_to_redis=push_to_redis,
                                      es_index=output_es_index,
                                      doc_type=output_doc_type)
                        result_Lst.clear()

        if result_Lst!=[]:
            if output_doc_type is None and output_es_index is None:
                output_result(result_Lst,
                              self.platform,
                              output_to_file=output_to_file,
                              filepath=filepath,
                              output_to_es_raw=output_to_es_raw,
                              output_to_es_register=output_to_es_register,
                              push_to_redis=push_to_redis)
            elif output_doc_type is not None and output_es_index is not None:
                output_result(result_Lst,
                              self.platform,
                              output_to_file=output_to_file,
                              filepath=filepath,
                              output_to_es_raw=output_to_es_raw,
                              output_to_es_register=output_to_es_register,
                              push_to_redis=push_to_redis,
                              es_index=output_es_index,
                              doc_type=output_doc_type)


    def doc_list(self, resultfile,donefile,doneurl,errorfile,area):
        list_data_lst=[]
        result=open(resultfile,'a')
        done=open(donefile,'a')
        donepage=open(doneurl,'a')
        error=open(errorfile,'a')
        count=1
        video_count=30
        while video_count==30 and count<=30:
            url='http://list.youku.com/category/show/c_84_a_'+area+'_s_1_d_1_p_'+str(count)+'.html'
            get_page=requests.get(url)
            get_page.encoding='utf-8'
            page = get_page.text
            print(url)
            soup = BeautifulSoup(page,'html.parser')
            midstep = soup.find_all('div',{'class':'p-thumb'})
            video_count=len(midstep)
            done.write(area)
            done.write(str(count))
            done.write('\n')
            done.flush()
            count+=1
            for line in midstep:
                url_step=line.a['href']
                if url_step[:4]!='http':
                    url='http:'+url_step
                else:
                    url=url_step
                try:
                    get_page=requests.get(url)
                    get_page.encoding='utf-8'
                    page = get_page.text
                    soup = BeautifulSoup(page,'html.parser')
                    print(url)
                    try:
                        donepage.write(url)
                        donepage.write('\n')
                        donepage.flush()
                        album_name=soup.find('h2').a.text
                        album_rating=soup.find('span',{'class':'score'}).text
                        sumvideo=soup.find('p',{'class':'tvintro'}).text
                        sumvideo_int=int(re.findall('\d+',sumvideo)[0])
                        if sumvideo_int<=100:
                            midstep=soup.find('div',{'id':'listitem_page1'})
                            first_video=midstep.find('div',{'class':'item item-cover current'})
                            url='http:'+first_video.a['href']
                            title=first_video['title']
                            dura=first_video.find('span',{'class':'c-time'}).text
                            dura_lst=dura.split(':')
                            if len(dura_lst)==1:
                                duration=int(dura_lst[0])
                            elif len(dura_lst)==2:
                                duration=int(dura_lst[0])*60+int(dura_lst[1])
                            elif len(dura_lst)==3:
                                duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                            playcount_str=first_video.find('div',{'class':'status'}).text
                            playcount_str=playcount_str.replace('次播放','')
                            playcount_str=playcount_str.replace(',','')
                            thousand_10='万'
                            if thousand_10 in playcount_str:
                                playcount_str=playcount_str.replace('万','')
                                play_count=int((float(playcount_str))*1e4)
                            else:
                                play_count=int(playcount_str)
                            fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                            D0={'url':url,'tilte':title,'duartion':duration,
                                'play_count':play_count,'album_name':album_name,
                                'album_rating':album_rating,'fetch_time':fetch_time}
                            D1={'url':url,'video_infor':D0}
                            list_data_lst.append(D1)
                            json_D1=json.dumps(D1)
                            result.write(json_D1)
                            result.write('\n')
                            result.flush()
                            print('get one line')
                            other_video=midstep.find_all('div',{'class':'item item-cover'})
                            for line in other_video:
                                url='http:'+first_video.a['href']
                                title=first_video['title']
                                dura=first_video.find('span',{'class':'c-time'}).text
                                dura_lst=dura.split(':')
                                if len(dura_lst)==1:
                                    duration=int(dura_lst[0])
                                elif len(dura_lst)==2:
                                    duration=int(dura_lst[0])*60+int(dura_lst[1])
                                elif len(dura_lst)==3:
                                    duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                                playcount_str=first_video.find('div',{'class':'status'}).text
                                playcount_str=playcount_str.replace('次播放','')
                                playcount_str=playcount_str.replace(',','')
                                thousand_10='万'
                                if thousand_10 in playcount_str:
                                    playcount_str=playcount_str.replace('万','')
                                    play_count=int((float(playcount_str))*1e4)
                                else:
                                    play_count=int(playcount_str)
                                fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                                D0={'url':url,'tilte':title,'duartion':duration,
                                'play_count':play_count,'album_name':album_name,
                                'album_rating':album_rating,'fetch_time':fetch_time}
                                D1={'url':url,'video_infor':D0}
                                list_data_lst.append(D1)
                                json_D1=json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                print('get one line')
                            result.flush()
                        if sumvideo_int>100:
                            if sumvideo_int%50==0:
                                page_num=int((sumvideo_int/50))
                            else:
                                page_num=int((sumvideo_int)/50)+1
                            videoid=re.findall('\d+',(re.findall('videoId:"[0-9]{1,10}"',page)[0]))[0]
                            showid=re.findall('\d+',(re.findall('showid:"[0-9]{1,10}"',page)[0]))[0]
                            url_lst=['http://v.youku.com/page/playlist?&l=debug&pm=3&vid='+videoid+
                                     '&fid=0&showid='+showid+'&sid=0&page={}&callback=tuijsonp23'
                                     .format(str(i)) for i in range(1,page_num+1)]
                            for line in url_lst:
                                get_page = requests.get(line)
                                get_page.encoding = 'utf-8'
                                page = get_page.text
                                page_dic = eval(page[34:-2])
                                videolab = page_dic['html']
                                videolab = videolab.replace('\\','')
                                soup = BeautifulSoup(videolab,'html.parser')
                                first_video=soup.find('div',{'class':'item item-cover current'})
                                url='http:'+first_video.a['href']
                                title=first_video['title']
                                dura=first_video.find('span',{'class':'c-time'}).text
                                dura_lst=dura.split(':')
                                if len(dura_lst)==1:
                                    duration=int(dura_lst[0])
                                elif len(dura_lst)==2:
                                    duration=int(dura_lst[0])*60+int(dura_lst[1])
                                elif len(dura_lst)==3:
                                    duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                                playcount_str=first_video.find('div',{'class':'status'}).text
                                playcount_str=playcount_str.replace('次播放','')
                                playcount_str=playcount_str.replace(',','')
                                thousand_10='万'
                                if thousand_10 in playcount_str:
                                    playcount_str=playcount_str.replace('万','')
                                    play_count=int((float(playcount_str))*1e4)
                                else:
                                    play_count=int(playcount_str)
                                fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                                D0={'url':url,'tilte':title,'duartion':duration,
                                    'play_count':play_count,'album_name':album_name,
                                    'album_rating':album_rating,'fetch_time':fetch_time}
                                D1={'url':url,'video_infor':D0}
                                list_data_lst.append(D1)
                                json_D1=json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print('get one line')
                                other_video=midstep.find_all('div',{'class':'item item-cover'})
                                for line in other_video:
                                    url='http:'+first_video.a['href']
                                    title=first_video['title']
                                    dura=first_video.find('span',{'class':'c-time'}).text
                                    dura_lst=dura.split(':')
                                    if len(dura_lst)==1:
                                        duration=int(dura_lst[0])
                                    elif len(dura_lst)==2:
                                        duration=int(dura_lst[0])*60+int(dura_lst[1])
                                    elif len(dura_lst)==3:
                                        duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                                    playcount_str=first_video.find('div',{'class':'status'}).text
                                    playcount_str=playcount_str.replace('次播放','')
                                    playcount_str=playcount_str.replace(',','')
                                    thousand_10='万'
                                    if thousand_10 in playcount_str:
                                        playcount_str=playcount_str.replace('万','')
                                        play_count=int((float(playcount_str))*1e4)
                                    else:
                                        play_count=int(playcount_str)
                                    fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                                    D0={'url':url,'tilte':title,'duartion':duration,'play_count':play_count,
                                        'album_name':album_name,'album_rating':album_rating,'fetch_time':fetch_time}
                                    D1={'url':url,'video_infor':D0}
                                    list_data_lst.append(D1)
                                    json_D1=json.dumps(D1)
                                    result.write(json_D1)
                                    result.write('\n')
                                    print('get one line')
                                result.flush()
                    except:
                        error.write(url)
                        error.write('\n')
                        error.flush()
                        print('there is an error')
                except:
                    error.write(url)
                    error.write('\n')
                    error.flush()
                    print('there is an error')
        done.close()
        result.close()
        return list_data_lst

    def add_infor(self,taskname,resultname):
        task=open(taskname,'a')
        result=open(resultname,'a')
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic=eval(line)
            video_infor=line_dic['video_infor']
            task_lst.append(video_infor)
        for line in task_lst:
            url=line['url']
            try:
                video_dic=self.video_page(url)
                line['releaser']=video_dic['releaser']
                line['release_time']=video_dic['release_time']
                line['releaserUrl']=video_dic['releaserUrl']
                line['tags']=video_dic['tags']
                json_line=json.dumps(line)
                result_lst.append(line)
                result.write(json_line)
                result.write('\n')
                result.flush()
            except:
                print('no video infor')
        result.close()
        return result_lst

    def get_doc(self,cluename,taskname,resultname,donename):
        task=open(taskname)
        result=open(resultname,'a')
        clue=open(cluename,'a')
        done=open(donename,'a')
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls=line['url']
            urls=urls.replace('\n','')
            album_name=line['album_name']
            album_rating=line['album_rating']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = (" ".join(re.findall('showid: &#\d+;\d+',page))).split(';')[1]
                videoId = (" ".join(re.findall('videoId: &#\d+;\d+',page))).split(';')[1]
                videoCategoryId = (" ".join(re.findall('videoCategoryId: &#\d+;\d+',page))).split(';')[1]
                componentid = (" ".join(re.findall('"componentId":\d+',page))).split(':')[1]
                D0={'showid':showid ,'videoid':videoId,'videocategoryid':videoCategoryId,'componentid':componentid}
                json_D0 = json.dumps(D0)
                clue.write(json_D0)
                clue.write('\n')
                clue.flush()
                count=1
                thousand_10='万'
                url='http://v.youku.com/page/playlist?&l=debug&pm=3&vid='+videoId+'&fid=0&showid='+showid+'&sid=0&componentid='+componentid+'&videoCategoryId='+videoCategoryId+'&isSimple=false&page='+str(count)
                get_page = requests.get(url)
                get_page.encoding = 'utf-8'
                page = get_page.text
                page_dic = eval(page)
                video_infor = page_dic['html']
                soup=BeautifulSoup(video_infor,'html.parser')
                video_dic = soup.find_all('div',{'class':'item item-cover'})
                sumofvideo = len(video_dic)
                for line in video_dic:
                    title=line['title']
                    url = line.a['href']
                    if 'http:' not in url:
                        url='http:'+url
                    dura=line.find('span',{'class':'c-time'}).text
                    dura_lst=dura.split(':')
                    if len(dura_lst)==1:
                        duration=int(dura_lst[0])
                    elif len(dura_lst)==2:
                        duration=int(dura_lst[0])*60+int(dura_lst[1])
                    elif len(dura_lst)==3:
                        duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                    play_count_str=line.find('div',{'class':'status'}).text
                    play_count_str=play_count_str.replace('次播放','')
                    play_count_str=play_count_str.replace(',','')
                    if thousand_10 in play_count_str:
                        play_count_str=play_count_str.replace('万','')
                        play_count=int((float(play_count_str))*1e4)
                    else:
                        play_count=int(play_count_str)
                    fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                    D1={'album_name':album_name,'album_rating':album_rating,'title':title,
                        'duration':duration,'play_count':play_count,'url':url,'fetch_time':fetch_time}
                    if D1 not in result_lst:
                        result_lst.append(D1)
                        json_D1 = json.dumps(D1)
                        result.write(json_D1)
                        result.write('\n')
                        result.flush()
                        print(url)
                while sumofvideo == 100:
                    count+=1
                    url='http://v.youku.com/page/playlist?&l=debug&pm=3&vid='+videoId+'&fid=0&showid='+showid+'&sid=0&componentid='+componentid+'&videoCategoryId='+videoCategoryId+'&isSimple=false&page='+str(count)
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    page_dic = eval(page)
                    video_infor = page_dic['html']
                    if video_infor =="\n":
                        sumofvideo=0
                    else:
                        soup=BeautifulSoup(video_infor,'html.parser')
                        video_dic = soup.find_all('div',{'class':'item item-cover'})
                        sumofvideo = len(video_dic)
                        for line in video_dic:
                            title=line['title']
                            url = line.a['href']
                            if 'http:' not in url:
                                url='http:'+url
                            dura=line.find('span',{'class':'c-time'}).text
                            dura_lst=dura.split(':')
                            if len(dura_lst)==1:
                                duration=int(dura_lst[0])
                            elif len(dura_lst)==2:
                                duration=int(dura_lst[0])*60+int(dura_lst[1])
                            elif len(dura_lst)==3:
                                duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                            play_count_str=line.find('div',{'class':'status'}).text
                            play_count_str=play_count_str.replace('次播放','')
                            play_count_str=play_count_str.replace(',','')
                            if thousand_10 in play_count_str:
                                play_count_str=play_count_str.replace('万','')
                                play_count=int((float(play_count_str))*1e4)
                            else:
                                play_count=int(play_count_str)
                            fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                            D1={'album_name':album_name,'album_rating':album_rating,'title':title,
                                'duration':duration,'play_count':play_count,'url':url,'fetch_time':fetch_time}
                            if D1 not in result_lst:
                                result_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print(url)
                done.write(urls)
                done.write('\n')
                done.flush()
            except:
                pass
        done.close()
        task.close()
        result.close()
        clue.close()
        return result_lst


    def one_video(self,taskname):
        task=open(taskname)
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls=line['url']
            urls=urls.replace('\n','')
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = (" ".join(re.findall('showid: &#\d+;\d+',page))).split(';')[1]
                print('more video')
            except:
                result_lst.append(urls)
                print('one video')
        task.close()
        return result_lst



    def get_releaser(self,taskname,resultname,errorname):
        task=open(taskname)
        result=open(resultname,'a')
        error=open(errorname,'a')
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls=line['url']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            page2 = page.replace('\\','')
            try:
                releaser=re.findall("[\u4e00-\u9fa5]+","".join(re.findall('class="" title="[\u4e00-\u9fa5]+"',page2)))[0]
                print('get releaser')
            except:
                releaser=None
                error.write(urls)
                error.write('\n')
                error.flush()
                print('no releaser')
            line['releaser']=releaser
            result_lst.append(line)
            json_line=json.dumps(line)
            result.write(json_line)
            result.write('\n')
            result.flush()
        result.close()
        return result_lst


    def get_release_time(self,url):
        get_page=requests.get(url)
        page=get_page.text
        soup = BeautifulSoup(page,'html.parser')
        try:
            time_one=soup.find('span',{'class':'bold mr3'}).text
        except AttributeError:
            time_one=None
        try:
            time_two=time_one[4:]
        except  TypeError:
            time_two=None
        try:
            midsteptime=time_two.strip()
        except  AttributeError:
            midsteptime=None
        try:
            release_time = int(datetime.datetime.strptime(midsteptime, '%Y-%m-%d').timestamp()*1e3)
        except  (TypeError,ValueError):
            release_time = None
        D_video_page={"release_time":release_time,'url':url}
        print('get one line')
        return D_video_page


    def get_video_play_count_releaser_info(self,url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page,'html.parser')
        try:
            video_info = soup.find('div',{'class':'item item-cover current'})
            play_count_str = video_info.find('div',{'class':'status'}).text
            play_count_str = play_count_str.replace('次播放','')
            if '万' in play_count_str:
                play_count_str = play_count_str.replace('万','')
                play_count = int(float(play_count_str)*1e4)
            else:
                play_count = int(play_count_str.replace(',',''))
        except:
            play_count = 0
        try:
            releaser = soup.find('span',{'id':"module_basic_sub"}).a.text
            releaser = releaser.replace('\n','')
            releaser = releaser.replace(' ','')
            releaserUrl = soup.find('span',{'id':"module_basic_sub"}).a['href']
            if releaserUrl[0:4] == 'http':
                pass
            else:
                releaserUrl = 'http:' + releaserUrl
        except:
            releaser = None
            releaserUrl = None
        D0 = {'play_count':play_count, 'releaser':releaser, 'releaserUrl':releaserUrl}
        return D0


    def get_release_time_from_str(self,rt_str):
        minute = '分钟'
        hour = '小时'
        yesterday = '昨天'
        the_day_before_yesterday = '前天'
        the_day_before_many_days = '天前'
        if minute in rt_str or hour in rt_str :
            year = datetime.date.today().year
            month = datetime.date.today().month
            day = datetime.date.today().day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        elif yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days = 1)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        elif the_day_before_yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days = 2)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        elif the_day_before_many_days in rt_str:
            day_shift = int(re.findall('\d+',rt_str)[0])
            raw_data = datetime.date.today() - datetime.timedelta(days = day_shift)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        else:
            release_time = 0
        return release_time



    def list_page_all(self,
                      output_to_file=False, filepath=None,
                      output_to_es_raw=False,
                      output_to_es_register=False,
                      push_to_redis=False,
                      page_num_max=25):
        """
        Get all videos in youku.com '片库' in the home page bottom and the '全部'
        category with '最新发布' filter.
        At present (2018-06-28) the url is http://list.youku.com/category/video/c_0_d_1_s_2.html?spm=a2h1n.8251846.selectID.5!2~5~5!2~1~3!2~A
        This url changes to the following format
        'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str(count) + '.html'
        where count is page number.
        """
        # if page_num_max argument is passed in a value greater than 25, will use 25,
        # at present (2018-06-28) there is only 25 pages at most.
        if page_num_max > 25:
            page_num_max = 25
        result_Lst = []
        page_num = 1
        while page_num <= page_num_max:
            lst_url = 'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str(page_num) + '.html'
            print('Getting data on page %d: %s, %s'
                  % (page_num, lst_url, datetime.datetime.now()))
            get_page = retry_get_url(lst_url)
            if get_page is None:
                print('Failed to get page url: %s, will continue to next page.'
                      % lst_url)
                continue
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_agg:
                title = line.find('li', {'class': 'title'}).text
                dura_str = line.find('li', {'class': 'status'}).text
                dura_lst = dura_str.split(':')
                if len(dura_lst) == 3:
                    duration = int(int(dura_lst[0])*3600 + int(dura_lst[1])*60 + int(dura_lst[2]))
                elif len(dura_lst) == 2:
                    duration = int(int(dura_lst[0])*60 + int(dura_lst[1]))
                else:
                    duration = 0
                url = line.a['href']
                if url[0:4] != 'http':
                    url = 'https:' + url
                video_info_dic = self.get_video_play_count_releaser_info(url)
                fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)

                rt_str = line.find('li', {'class': ' '}).text
                release_time = self.get_release_time_from_str(rt_str)

                self.video_data['title'] = title
                self.video_data['duration'] = duration
                self.video_data['url'] = url
                self.video_data['play_count'] = video_info_dic['play_count']
                self.video_data['releaser'] = video_info_dic['releaser']
                self.video_data['releaserUrl'] = video_info_dic['releaserUrl']
                self.video_data['release_time'] = release_time
                self.video_data['fetch_time'] = fetch_time

                get_data = copy.deepcopy(self.video_data)
                result_Lst.append(get_data)

                if len(result_Lst) >= 100:
                    print('Got %d lines, %s' % (len(result_Lst),
                                                datetime.datetime.now()))
                    output_result(result_Lst, self.platform,
                                  output_to_file=output_to_file,
                                  filepath=filepath,
                                  output_to_es_raw=output_to_es_raw,
                                  output_to_es_register=output_to_es_register,
                                  push_to_redis=push_to_redis)
                    result_Lst.clear()

            page_num += 1

        if result_Lst != []:
            print('Got %d lines, %s' % (len(result_Lst),
                                        datetime.datetime.now()))
            output_result(result_Lst, self.platform,
                          output_to_file=output_to_file,
                          filepath=filepath,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          push_to_redis=push_to_redis)
            result_Lst.clear()

    def get_title_subtitle(self, url):
        get_page = retry_get_url(url)
        if get_page is None:
            print('Failed to get url: %s' % url)
            return
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        try:
            subtitle = soup.find('span', {'id': 'subtitle'}).text
        except:
            subtitle = None
        try:
            title = soup.h1.find('a', {'target':'_blank'}).text
            title_str = title + subtitle
        except:
            title_str = subtitle
        return title_str

    def select_tv_series_video(self, sp_video_album_name,
                               result_file=None, # modify to from es
                               output_to_file=False,
                               filepath=None,
                               output_to_es_raw=False,
                               output_to_es_register=False,
                               push_to_redis=False):
        select_result = []
        meta = Meta()
        result_lst = meta.dic_file_to_lst(result_file)
        for line in result_lst:
            url = line['url']
            title_str = self.get_title_subtitle(url)
            if sp_video_album_name in title_str:
                line.update({'title': title_str})
                select_result.append(line)
                print('it_belongs_to_what_we_want')
                if len(select_result) >= 100:
                    print('Got %d lines, %s' % (len(select_result), datetime.datetime.now()))
                    output_result(select_result, self.platform,
                                  output_to_file=output_to_file,
                                  filepath=filepath,
                                  output_to_es_raw=output_to_es_raw,
                                  output_to_es_register=output_to_es_register,
                                  push_to_redis=push_to_redis)
                    select_result.clear()
            else:
                print('not_belongs_to')
        if select_result != []:
            print('Got %d lines, %s' % (len(select_result),datetime.datetime.now()))
            output_result(select_result, self.platform,
                          output_to_file=output_to_file,
                          filepath=filepath,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          push_to_redis=push_to_redis)
            select_result.clear()



#list page asynchronous crawler
    def start_list_page(self, task_list,
                        output_to_file=False,
                        filepath=None,
                        output_to_es_raw=False,
                        push_to_redis=False,
                        es_index='crawler-data-raw',
                        doc_type='doc',
                        output_to_es_register=False,):
        self.list_page_task(task_list)
        self.download_list_page_single_process()


    def list_page_task(self, task_list, page_num_max=25):
        TASK_LIST = []
        for line in task_list:
            for num in range(1, page_num_max+1):
                url = line.replace('fangyucheng', str(num))
                TASK_LIST.append(url)
        random.shuffle(TASK_LIST)
        connect_with_redis.push_list_url_to_redis(platform=self.platform,
                                                  result_lst=TASK_LIST)

    async def download_page(self, session, url):
        get_page = await session.get(url)
        page = await get_page.text("utf-8", errors="ignore")
        return page

    async def download_list_page(self, loop):
        task_list = connect_with_redis.retrieve_list_url_from_redis(platform=self.platform)
        async with aiohttp.ClientSession() as session:
            task = [loop.create_task(self.download_page(session, url)) for url in task_list]
            done, pending = await asyncio.wait(task)
            RESULT_LIST = [d.result() for d in done]
            connect_with_redis.push_list_page_html_to_redis(platform=self.platform,
                                                            result_lst=RESULT_LIST)

    def download_list_page_single_process(self):
        key = 'youku_list_url'
        while connect_with_redis.length_of_lst(key) > 0:
            loop = asyncio.get_event_loop()
            loop.run_until_complete(self.download_list_page(loop))

    def parse_list_page_single_process(self):
        pid = os.getpid()
        URL_LIST = []
        key = 'youku_list_page_html'
        while connect_with_redis.length_of_lst(key) > 0:
            PAGE = connect_with_redis.retrieve_list_page_html_from_redis(platform=self.platform)
            soup = BeautifulSoup(PAGE, 'html.parser')
            video_url_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_url_agg:
                url = line.a['href']
                video_id_str = ' '.join(re.findall('id_.*html', url))
                video_id = video_id_str.replace('id_', '')
                tudou_url = 'https://video.tudou.com/v/%s' % video_id
                URL_LIST.append(tudou_url)
                connect_with_redis.push_video_url_to_redis_set(platform='new_tudou',
                                                               url_lst=URL_LIST)
            print("platform: %s, action: parse list page, process_id: %s, get %s urls"
                  % (self.platform, pid, len(URL_LIST)))

    def parse_list_page_multi_process(self, processes_num=20):
        pool = Pool(processes=processes_num)
        for num in range(processes_num):
            pool.apply_async(self.parse_list_page_single_process)
        pool.close()
        pool.join()


"""
due to the video play_count in new_tudou is equal to youku, and it is earier to get
play_count in new_tudou, I rebuild the youku video url so that I can use new_tudou
crawler to get data
"""
#    async def download_video_page(self, loop):
#        task_list = connect_with_redis.retrieve_video_url_from_redis_set(platform=self.platform)
#        async with aiohttp.ClientSession() as session:
#            task = [loop.create_task(self.download_page(session, url)) for url in task_list]
#            done, pending = await asyncio.wait(task)
#            RESULT_LIST = [d.result() for d in done]
#            connect_with_redis.push_video_page_html_to_redis(platform=self.platform,
#                                                             result_lst=RESULT_LIST)
#    def download_video_page_async_single_process(self):
#        key = 'youku_video_url'
#        pid = os.getpid()
#        while connect_with_redis.length_of_set(key) > 0:
#            loop = asyncio.get_event_loop()
#            loop.run_until_complete(self.download_video_page(loop))
#            print("platform: %s, action: download video page, process_id: %s"
#                  % (self.platform, pid))
#
#    def download_video_page_async_multi_process(self, processes_num=10):
#        pool = Pool(processes=processes_num)
#        for num in range(processes_num):
#            pool.apply_async(self.download_video_page_async_single_process)
#        pool.close()
#        pool.join()
#
#    def parse_video_page(self, resp_str):
#        soup = BeautifulSoup(resp_str, 'html.parser')
#        title = soup.find('meta', {'name': 'title'})['content']
#        url = soup.find('meta', {'itemprop': 'url'})['content']
#        release_time_str = soup.find('meta', {'itemprop': 'datePublished'})['content']
#        release_time = int(datetime.datetime.strptime(release_time_str,
#                                                      "%Y-%m-%d %H:%M:%S").timestamp()*1e3)
#        releaser = soup.find('span',{'id': "module_basic_sub"}).a.text
#        releaser = releaser.replace('\n', '')
#        releaser = releaser.replace(' ', '')
#        duration_str =  ' '.join(re.findall("seconds: '.*'", resp_str))
#        if '.' in duration_str:
#            duration_str = ' '.join(re.findall("seconds: '\d+.", duration_str))
#            duration = int(' '.join(re.findall('\d+', duration_str))) + 1
#        else:
#            duration = int(' '.join(re.findall('\d+', duration_str)))
#        releaserUrl = soup.find('span',{'id': "module_basic_sub"}).a['href']
#        if releaserUrl[0:4] != 'http':
#            releaserUrl = 'http:' + releaserUrl
#        try:
#            video_info = soup.find('div',{'class': 'item item-cover current'})
#            play_count_str = video_info.find('div',{'class': 'status'}).text
#            play_count = trans_play_count(play_count_str)
#        except:
#            print("can't find play count at %s" % url)
#            play_count = 0
#        try:
#            releaser = soup.find('span',{'id':"module_basic_sub"}).a.text
#            releaserUrl = soup.find('span',{'id':"module_basic_sub"}).a['href']
#        except:
#            releaser = None
#            releaserUrl = None
#        else:
#            releaser = releaser.replace('\n','')
#            releaser = releaser.replace(' ','')
#            if releaserUrl[0:4] != 'http':
#                releaserUrl = 'http:' + releaserUrl
#        fetch_time = int(time.time()*1e3)
#        video_info = {"title": title,
#                      'url': url,
#                      'duration': duration,
#                      'releaser': releaser,
#                      'play_count': play_count,
#                      'releaserUrl': releaserUrl,
#                      'release_time': release_time,
#                      'fetch_time': fetch_time}
#        return video_info
#
#    def parse_video_page_single_process(self, output_to_file=False,
#                                        filepath=None,
#                                        push_to_redis=False,
#                                        output_to_es_raw=False,
#                                        es_index=None,
#                                        doc_type=None,
#                                        output_to_es_register=False):
#        pid = os.getpid()
#        RESULT_LIST = []
#        key = 'youku_video_page_html'
#        while connect_with_redis.length_of_lst(key) > 0:
#            VIDEO_PAGE_HTML = connect_with_redis.retrieve_video_page_html_from_redis(platform=self.platform)
#            try:
#                video_info = self.parse_video_page(resp_str=VIDEO_PAGE_HTML)
#            except:
#                video_info = None
#                print("failed to parse video info")
#            if video_info is not None:
#                RESULT_LIST.append(video_info)
#                print("platform: %s, action: parse video page, process_id: %s, has done %s"
#                      % (self.platform, pid, len(RESULT_LIST)))
#                if len(RESULT_LIST) >= 100:
#                    output_result(result_Lst=RESULT_LIST,
#                                  platform=self.platform,
#                                  output_to_file=output_to_file,
#                                  filepath=filepath,
#                                  push_to_redis=push_to_redis,
#                                  output_to_es_raw=output_to_es_raw,
#                                  es_index=es_index,
#                                  doc_type=doc_type,
#                                  output_to_es_register=output_to_es_register)
#                    RESULT_LIST.clear()
#        if len(RESULT_LIST) != []:
#            output_result(result_Lst=RESULT_LIST,
#                          platform=self.platform,
#                          output_to_file=output_to_file,
#                          filepath=filepath,
#                          push_to_redis=push_to_redis,
#                          output_to_es_raw=output_to_es_raw,
#                          es_index=es_index,
#                          doc_type=doc_type,
#                          output_to_es_register=output_to_es_register)
#            RESULT_LIST.clear()
#
#    def parse_video_page_multi_process(self, para_dic,
#                                       processes_num=30):
#        pool = Pool(processes=processes_num)
#        for num in range(processes_num):
#            pool.apply_async(self.parse_video_page_single_process, kwds=para_dic)
#        pool.close()
#        pool.join()

# test
if __name__=='__main__':
    test = Crawler_youku()
    vid = 'XMjc5MTQyNjg0OA'
    test_selenium = test.get_video_play_count_by_vid(vid)
    sr_yk = test.search_page(keyword='任正非 BBC', search_pages_max=2)

d40fdc7df97f05ac69b73a57104dd6835dfafe65
<mask token>


class Crawler_youku:
    <mask token>
    <mask token>

    def get_video_infor_by_network(self, url_Lst):
        video_id_Lst = []
        for url in url_Lst:
            video_id = ' '.join(re.findall('id.*html', url)).split('_')[1
                ].split('.')[0]
            video_id_Lst.append(video_id)
    <mask token>

    def get_video_play_count_by_vid(self, vid):
        mobile_url = 'https://m.youku.com/video/id_' + vid + '==.html'
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        time.sleep(10)
        title = browser.find_element_by_class_name('online').text
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',', '')
        pc_midstep4 = re.findall('[0-9]+', pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0] * 10000.0 + pc_midstep4[1] * 1000.0
        else:
            play_count = ' '.join(pc_midstep4)
        browser.close()
        D0 = {'title': title, 'play_count': play_count}
        return D0
    <mask token>
    <mask token>
    <mask token>

    def get_releaser_detail(self, url):
        releaser_data_Lst = []
        get_page = requests.get(url)
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('a', {'class': 'username'})['title']
        releaser_intro = soup.find('span', {'class': 'desc'}).text
        total_play_count = soup.find('li', {'class': 'vnum'})['title']
        fans = soup.find('li', {'class': 'snum'})['title']
        midstep = soup.find_all('div', {'class': 'title'})
        for brother in midstep:
            if brother.h3.text == '视频':
                num = brother.find('span').text
                self.TotalVideo_num = num[1:-1]
        D_releaser_page = {'releaser': releaser, 'releaser_intro':
            releaser_intro, 'total_play_count': total_play_count, 'follow':
            fans, 'TotalVideo_num': self.TotalVideo_num}
        releaser_data_Lst.append(D_releaser_page)
        return D_releaser_page

    def get_releaser_info(self, releaserUrl):
        get_page = retry_get_url(releaserUrl)
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('a', {'class': 'username'})['title']
        releaser_id = soup.find('a', {'class': 'user-avatar'})['href']
        D0 = {'releaser': releaser, 'releaser_id': releaser_id}
        return D0
    <mask token>

    def doc_list(self, resultfile, donefile, doneurl, errorfile, area):
        list_data_lst = []
        result = open(resultfile, 'a')
        done = open(donefile, 'a')
        donepage = open(doneurl, 'a')
        error = open(errorfile, 'a')
        count = 1
        video_count = 30
        while video_count == 30 and count <= 30:
            url = ('http://list.youku.com/category/show/c_84_a_' + area +
                '_s_1_d_1_p_' + str(count) + '.html')
            get_page = requests.get(url)
            get_page.encoding = 'utf-8'
            page = get_page.text
            print(url)
            soup = BeautifulSoup(page, 'html.parser')
            midstep = soup.find_all('div', {'class': 'p-thumb'})
            video_count = len(midstep)
            done.write(area)
            done.write(str(count))
            done.write('\n')
            done.flush()
            count += 1
            for line in midstep:
                url_step = line.a['href']
                if url_step[:4] != 'http':
                    url = 'http:' + url_step
                else:
                    url = url_step
                try:
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    soup = BeautifulSoup(page, 'html.parser')
                    print(url)
                    try:
                        donepage.write(url)
                        donepage.write('\n')
                        donepage.flush()
                        album_name = soup.find('h2').a.text
                        album_rating = soup.find('span', {'class': 'score'}
                            ).text
                        sumvideo = soup.find('p', {'class': 'tvintro'}).text
                        sumvideo_int = int(re.findall('\\d+', sumvideo)[0])
                        if sumvideo_int <= 100:
                            midstep = soup.find('div', {'id': 'listitem_page1'}
                                )
                            first_video = midstep.find('div', {'class':
                                'item item-cover current'})
                            url = 'http:' + first_video.a['href']
                            title = first_video['title']
                            dura = first_video.find('span', {'class': 'c-time'}
                                ).text
                            dura_lst = dura.split(':')
                            if len(dura_lst) == 1:
                                duration = int(dura_lst[0])
                            elif len(dura_lst) == 2:
                                duration = int(dura_lst[0]) * 60 + int(dura_lst
                                    [1])
                            elif len(dura_lst) == 3:
                                duration = int(dura_lst[0]) * 3600 + int(
                                    dura_lst[1]) * 60 + int(dura_lst[2])
                            playcount_str = first_video.find('div', {
                                'class': 'status'}).text
                            playcount_str = playcount_str.replace('次播放', '')
                            playcount_str = playcount_str.replace(',', '')
                            thousand_10 = '万'
                            if thousand_10 in playcount_str:
                                playcount_str = playcount_str.replace('万', '')
                                play_count = int(float(playcount_str) * 10000.0
                                    )
                            else:
                                play_count = int(playcount_str)
                            fetch_time = int(datetime.datetime.timestamp(
                                datetime.datetime.now()) * 1000.0)
                            D0 = {'url': url, 'tilte': title, 'duartion':
                                duration, 'play_count': play_count,
                                'album_name': album_name, 'album_rating':
                                album_rating, 'fetch_time': fetch_time}
                            D1 = {'url': url, 'video_infor': D0}
                            list_data_lst.append(D1)
                            json_D1 = json.dumps(D1)
                            result.write(json_D1)
                            result.write('\n')
                            result.flush()
                            print('get one line')
                            other_video = midstep.find_all('div', {'class':
                                'item item-cover'})
                            for line in other_video:
                                url = 'http:' + first_video.a['href']
                                title = first_video['title']
                                dura = first_video.find('span', {'class':
                                    'c-time'}).text
                                dura_lst = dura.split(':')
                                if len(dura_lst) == 1:
                                    duration = int(dura_lst[0])
                                elif len(dura_lst) == 2:
                                    duration = int(dura_lst[0]) * 60 + int(
                                        dura_lst[1])
                                elif len(dura_lst) == 3:
                                    duration = int(dura_lst[0]) * 3600 + int(
                                        dura_lst[1]) * 60 + int(dura_lst[2])
                                playcount_str = first_video.find('div', {
                                    'class': 'status'}).text
                                playcount_str = playcount_str.replace('次播放', ''
                                    )
                                playcount_str = playcount_str.replace(',', '')
                                thousand_10 = '万'
                                if thousand_10 in playcount_str:
                                    playcount_str = playcount_str.replace('万',
                                        '')
                                    play_count = int(float(playcount_str) *
                                        10000.0)
                                else:
                                    play_count = int(playcount_str)
                                fetch_time = int(datetime.datetime.
                                    timestamp(datetime.datetime.now()) * 1000.0
                                    )
                                D0 = {'url': url, 'tilte': title,
                                    'duartion': duration, 'play_count':
                                    play_count, 'album_name': album_name,
                                    'album_rating': album_rating,
                                    'fetch_time': fetch_time}
                                D1 = {'url': url, 'video_infor': D0}
                                list_data_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                print('get one line')
                            result.flush()
                        if sumvideo_int > 100:
                            if sumvideo_int % 50 == 0:
                                page_num = int(sumvideo_int / 50)
                            else:
                                page_num = int(sumvideo_int / 50) + 1
                            videoid = re.findall('\\d+', re.findall(
                                'videoId:"[0-9]{1,10}"', page)[0])[0]
                            showid = re.findall('\\d+', re.findall(
                                'showid:"[0-9]{1,10}"', page)[0])[0]
                            url_lst = [(
                                'http://v.youku.com/page/playlist?&l=debug&pm=3&vid='
                                 + videoid + '&fid=0&showid=' + showid +
                                '&sid=0&page={}&callback=tuijsonp23'.format
                                (str(i))) for i in range(1, page_num + 1)]
                            for line in url_lst:
                                get_page = requests.get(line)
                                get_page.encoding = 'utf-8'
                                page = get_page.text
                                page_dic = eval(page[34:-2])
                                videolab = page_dic['html']
                                videolab = videolab.replace('\\', '')
                                soup = BeautifulSoup(videolab, 'html.parser')
                                first_video = soup.find('div', {'class':
                                    'item item-cover current'})
                                url = 'http:' + first_video.a['href']
                                title = first_video['title']
                                dura = first_video.find('span', {'class':
                                    'c-time'}).text
                                dura_lst = dura.split(':')
                                if len(dura_lst) == 1:
                                    duration = int(dura_lst[0])
                                elif len(dura_lst) == 2:
                                    duration = int(dura_lst[0]) * 60 + int(
                                        dura_lst[1])
                                elif len(dura_lst) == 3:
                                    duration = int(dura_lst[0]) * 3600 + int(
                                        dura_lst[1]) * 60 + int(dura_lst[2])
                                playcount_str = first_video.find('div', {
                                    'class': 'status'}).text
                                playcount_str = playcount_str.replace('次播放', ''
                                    )
                                playcount_str = playcount_str.replace(',', '')
                                thousand_10 = '万'
                                if thousand_10 in playcount_str:
                                    playcount_str = playcount_str.replace('万',
                                        '')
                                    play_count = int(float(playcount_str) *
                                        10000.0)
                                else:
                                    play_count = int(playcount_str)
                                fetch_time = int(datetime.datetime.
                                    timestamp(datetime.datetime.now()) * 1000.0
                                    )
                                D0 = {'url': url, 'tilte': title,
                                    'duartion': duration, 'play_count':
                                    play_count, 'album_name': album_name,
                                    'album_rating': album_rating,
                                    'fetch_time': fetch_time}
                                D1 = {'url': url, 'video_infor': D0}
                                list_data_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print('get one line')
                                other_video = midstep.find_all('div', {
                                    'class': 'item item-cover'})
                                for line in other_video:
                                    url = 'http:' + first_video.a['href']
                                    title = first_video['title']
                                    dura = first_video.find('span', {
                                        'class': 'c-time'}).text
                                    dura_lst = dura.split(':')
                                    if len(dura_lst) == 1:
                                        duration = int(dura_lst[0])
                                    elif len(dura_lst) == 2:
                                        duration = int(dura_lst[0]) * 60 + int(
                                            dura_lst[1])
                                    elif len(dura_lst) == 3:
                                        duration = int(dura_lst[0]) * 3600 + int(
                                            dura_lst[1]) * 60 + int(dura_lst[2])
                                    playcount_str = first_video.find('div',
                                        {'class': 'status'}).text
                                    playcount_str = playcount_str.replace('次播放'
                                        , '')
                                    playcount_str = playcount_str.replace(',',
                                        '')
                                    thousand_10 = '万'
                                    if thousand_10 in playcount_str:
                                        playcount_str = playcount_str.replace('万',
                                            '')
                                        play_count = int(float(playcount_str) *
                                            10000.0)
                                    else:
                                        play_count = int(playcount_str)
                                    fetch_time = int(datetime.datetime.
                                        timestamp(datetime.datetime.now()) *
                                        1000.0)
                                    D0 = {'url': url, 'tilte': title,
                                        'duartion': duration, 'play_count':
                                        play_count, 'album_name':
                                        album_name, 'album_rating':
                                        album_rating, 'fetch_time': fetch_time}
                                    D1 = {'url': url, 'video_infor': D0}
                                    list_data_lst.append(D1)
                                    json_D1 = json.dumps(D1)
                                    result.write(json_D1)
                                    result.write('\n')
                                    print('get one line')
                                result.flush()
                    except:
                        error.write(url)
                        error.write('\n')
                        error.flush()
                        print('there is an error')
                except:
                    error.write(url)
                    error.write('\n')
                    error.flush()
                    print('there is an error')
        done.close()
        result.close()
        return list_data_lst

    def add_infor(self, taskname, resultname):
        task = open(taskname, 'a')
        result = open(resultname, 'a')
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            video_infor = line_dic['video_infor']
            task_lst.append(video_infor)
        for line in task_lst:
            url = line['url']
            try:
                video_dic = self.video_page(url)
                line['releaser'] = video_dic['releaser']
                line['release_time'] = video_dic['release_time']
                line['releaserUrl'] = video_dic['releaserUrl']
                line['tags'] = video_dic['tags']
                json_line = json.dumps(line)
                result_lst.append(line)
                result.write(json_line)
                result.write('\n')
                result.flush()
            except:
                print('no video infor')
        result.close()
        return result_lst

    def get_doc(self, cluename, taskname, resultname, donename):
        task = open(taskname)
        result = open(resultname, 'a')
        clue = open(cluename, 'a')
        done = open(donename, 'a')
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls = line['url']
            urls = urls.replace('\n', '')
            album_name = line['album_name']
            album_rating = line['album_rating']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = ' '.join(re.findall('showid: &#\\d+;\\d+', page)
                    ).split(';')[1]
                videoId = ' '.join(re.findall('videoId: &#\\d+;\\d+', page)
                    ).split(';')[1]
                videoCategoryId = ' '.join(re.findall(
                    'videoCategoryId: &#\\d+;\\d+', page)).split(';')[1]
                componentid = ' '.join(re.findall('"componentId":\\d+', page)
                    ).split(':')[1]
                D0 = {'showid': showid, 'videoid': videoId,
                    'videocategoryid': videoCategoryId, 'componentid':
                    componentid}
                json_D0 = json.dumps(D0)
                clue.write(json_D0)
                clue.write('\n')
                clue.flush()
                count = 1
                thousand_10 = '万'
                url = (
                    'http://v.youku.com/page/playlist?&l=debug&pm=3&vid=' +
                    videoId + '&fid=0&showid=' + showid +
                    '&sid=0&componentid=' + componentid +
                    '&videoCategoryId=' + videoCategoryId +
                    '&isSimple=false&page=' + str(count))
                get_page = requests.get(url)
                get_page.encoding = 'utf-8'
                page = get_page.text
                page_dic = eval(page)
                video_infor = page_dic['html']
                soup = BeautifulSoup(video_infor, 'html.parser')
                video_dic = soup.find_all('div', {'class': 'item item-cover'})
                sumofvideo = len(video_dic)
                for line in video_dic:
                    title = line['title']
                    url = line.a['href']
                    if 'http:' not in url:
                        url = 'http:' + url
                    dura = line.find('span', {'class': 'c-time'}).text
                    dura_lst = dura.split(':')
                    if len(dura_lst) == 1:
                        duration = int(dura_lst[0])
                    elif len(dura_lst) == 2:
                        duration = int(dura_lst[0]) * 60 + int(dura_lst[1])
                    elif len(dura_lst) == 3:
                        duration = int(dura_lst[0]) * 3600 + int(dura_lst[1]
                            ) * 60 + int(dura_lst[2])
                    play_count_str = line.find('div', {'class': 'status'}).text
                    play_count_str = play_count_str.replace('次播放', '')
                    play_count_str = play_count_str.replace(',', '')
                    if thousand_10 in play_count_str:
                        play_count_str = play_count_str.replace('万', '')
                        play_count = int(float(play_count_str) * 10000.0)
                    else:
                        play_count = int(play_count_str)
                    fetch_time = int(datetime.datetime.timestamp(datetime.
                        datetime.now()) * 1000.0)
                    D1 = {'album_name': album_name, 'album_rating':
                        album_rating, 'title': title, 'duration': duration,
                        'play_count': play_count, 'url': url, 'fetch_time':
                        fetch_time}
                    if D1 not in result_lst:
                        result_lst.append(D1)
                        json_D1 = json.dumps(D1)
                        result.write(json_D1)
                        result.write('\n')
                        result.flush()
                        print(url)
                while sumofvideo == 100:
                    count += 1
                    url = (
                        'http://v.youku.com/page/playlist?&l=debug&pm=3&vid=' +
                        videoId + '&fid=0&showid=' + showid +
                        '&sid=0&componentid=' + componentid +
                        '&videoCategoryId=' + videoCategoryId +
                        '&isSimple=false&page=' + str(count))
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    page_dic = eval(page)
                    video_infor = page_dic['html']
                    if video_infor == '\n':
                        sumofvideo = 0
                    else:
                        soup = BeautifulSoup(video_infor, 'html.parser')
                        video_dic = soup.find_all('div', {'class':
                            'item item-cover'})
                        sumofvideo = len(video_dic)
                        for line in video_dic:
                            title = line['title']
                            url = line.a['href']
                            if 'http:' not in url:
                                url = 'http:' + url
                            dura = line.find('span', {'class': 'c-time'}).text
                            dura_lst = dura.split(':')
                            if len(dura_lst) == 1:
                                duration = int(dura_lst[0])
                            elif len(dura_lst) == 2:
                                duration = int(dura_lst[0]) * 60 + int(dura_lst
                                    [1])
                            elif len(dura_lst) == 3:
                                duration = int(dura_lst[0]) * 3600 + int(
                                    dura_lst[1]) * 60 + int(dura_lst[2])
                            play_count_str = line.find('div', {'class':
                                'status'}).text
                            play_count_str = play_count_str.replace('次播放', '')
                            play_count_str = play_count_str.replace(',', '')
                            if thousand_10 in play_count_str:
                                play_count_str = play_count_str.replace('万', ''
                                    )
                                play_count = int(float(play_count_str) * 
                                    10000.0)
                            else:
                                play_count = int(play_count_str)
                            fetch_time = int(datetime.datetime.timestamp(
                                datetime.datetime.now()) * 1000.0)
                            D1 = {'album_name': album_name, 'album_rating':
                                album_rating, 'title': title, 'duration':
                                duration, 'play_count': play_count, 'url':
                                url, 'fetch_time': fetch_time}
                            if D1 not in result_lst:
                                result_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print(url)
                done.write(urls)
                done.write('\n')
                done.flush()
            except:
                pass
        done.close()
        task.close()
        result.close()
        clue.close()
        return result_lst

    def one_video(self, taskname):
        task = open(taskname)
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls = line['url']
            urls = urls.replace('\n', '')
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = ' '.join(re.findall('showid: &#\\d+;\\d+', page)
                    ).split(';')[1]
                print('more video')
            except:
                result_lst.append(urls)
                print('one video')
        task.close()
        return result_lst
    <mask token>
    <mask token>
    <mask token>
    <mask token>

    def list_page_all(self, output_to_file=False, filepath=None,
        output_to_es_raw=False, output_to_es_register=False, push_to_redis=
        False, page_num_max=25):
        """
        Get all videos in youku.com '片库' in the home page bottom and the '全部'
        category with '最新发布' filter.
        At present (2018-06-28) the url is http://list.youku.com/category/video/c_0_d_1_s_2.html?spm=a2h1n.8251846.selectID.5!2~5~5!2~1~3!2~A
        This url changes to the following format
        'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str(count) + '.html'
        where count is page number.
        """
        if page_num_max > 25:
            page_num_max = 25
        result_Lst = []
        page_num = 1
        while page_num <= page_num_max:
            lst_url = (
                'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str
                (page_num) + '.html')
            print('Getting data on page %d: %s, %s' % (page_num, lst_url,
                datetime.datetime.now()))
            get_page = retry_get_url(lst_url)
            if get_page is None:
                print(
                    'Failed to get page url: %s, will continue to next page.' %
                    lst_url)
                continue
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_agg:
                title = line.find('li', {'class': 'title'}).text
                dura_str = line.find('li', {'class': 'status'}).text
                dura_lst = dura_str.split(':')
                if len(dura_lst) == 3:
                    duration = int(int(dura_lst[0]) * 3600 + int(dura_lst[1
                        ]) * 60 + int(dura_lst[2]))
                elif len(dura_lst) == 2:
                    duration = int(int(dura_lst[0]) * 60 + int(dura_lst[1]))
                else:
                    duration = 0
                url = line.a['href']
                if url[0:4] != 'http':
                    url = 'https:' + url
                video_info_dic = self.get_video_play_count_releaser_info(url)
                fetch_time = int(datetime.datetime.timestamp(datetime.
                    datetime.now()) * 1000.0)
                rt_str = line.find('li', {'class': ' '}).text
                release_time = self.get_release_time_from_str(rt_str)
                self.video_data['title'] = title
                self.video_data['duration'] = duration
                self.video_data['url'] = url
                self.video_data['play_count'] = video_info_dic['play_count']
                self.video_data['releaser'] = video_info_dic['releaser']
                self.video_data['releaserUrl'] = video_info_dic['releaserUrl']
                self.video_data['release_time'] = release_time
                self.video_data['fetch_time'] = fetch_time
                get_data = copy.deepcopy(self.video_data)
                result_Lst.append(get_data)
                if len(result_Lst) >= 100:
                    print('Got %d lines, %s' % (len(result_Lst), datetime.
                        datetime.now()))
                    output_result(result_Lst, self.platform, output_to_file
                        =output_to_file, filepath=filepath,
                        output_to_es_raw=output_to_es_raw,
                        output_to_es_register=output_to_es_register,
                        push_to_redis=push_to_redis)
                    result_Lst.clear()
            page_num += 1
        if result_Lst != []:
            print('Got %d lines, %s' % (len(result_Lst), datetime.datetime.
                now()))
            output_result(result_Lst, self.platform, output_to_file=
                output_to_file, filepath=filepath, output_to_es_raw=
                output_to_es_raw, output_to_es_register=
                output_to_es_register, push_to_redis=push_to_redis)
            result_Lst.clear()
    <mask token>
    <mask token>
    <mask token>

    def list_page_task(self, task_list, page_num_max=25):
        TASK_LIST = []
        for line in task_list:
            for num in range(1, page_num_max + 1):
                url = line.replace('fangyucheng', str(num))
                TASK_LIST.append(url)
        random.shuffle(TASK_LIST)
        connect_with_redis.push_list_url_to_redis(platform=self.platform,
            result_lst=TASK_LIST)

    async def download_page(self, session, url):
        get_page = await session.get(url)
        page = await get_page.text('utf-8', errors='ignore')
        return page

    async def download_list_page(self, loop):
        task_list = connect_with_redis.retrieve_list_url_from_redis(platform
            =self.platform)
        async with aiohttp.ClientSession() as session:
            task = [loop.create_task(self.download_page(session, url)) for
                url in task_list]
            done, pending = await asyncio.wait(task)
            RESULT_LIST = [d.result() for d in done]
            connect_with_redis.push_list_page_html_to_redis(platform=self.
                platform, result_lst=RESULT_LIST)
    <mask token>
    <mask token>
    <mask token>


<mask token>

<mask token>


class Crawler_youku:

    def __init__(self, timeout=None, platform='youku'):
        if timeout == None:
            self.timeout = 10
        else:
            self.timeout = timeout
        self.platform = platform
        std_fields = Std_fields_video()
        self.video_data = std_fields.video_data
        self.video_data['platform'] = self.platform
        pop_key_Lst = ['channel', 'describe', 'repost_count', 'isOriginal',
            'video_id']
        for popk in pop_key_Lst:
            self.video_data.pop(popk)
        self.legal_channels = ['全部']
        self.list_page_url_dict = {'全部': '_'}

    def video_page(self, url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('span', {'id': 'module_basic_sub'}).a.text
        releaser = releaser.replace('\n', '')
        releaser = releaser.replace(' ', '')
        releaserUrl = soup.find('span', {'id': 'module_basic_sub'}).a['href']
        if releaserUrl[0:4] == 'http':
            pass
        else:
            releaserUrl = 'http:' + releaserUrl
        try:
            rt_str = soup.find('span', {'class': 'video-status'}).text
            rt_lst = re.findall('\\d+', rt_str)
            rt_to_timestamp = str(rt_lst[0]) + '-' + str(rt_lst[1]
                ) + '-' + str(rt_lst[2])
            release_time = int(datetime.datetime.strptime(rt_to_timestamp,
                '%Y-%m-%d').timestamp() * 1000.0)
        except:
            release_time = 0
        D0 = {'releaser': releaser, 'releaserUrl': releaserUrl,
            'release_time': release_time}
        return D0

    def get_video_infor_by_network(self, url_Lst):
        video_id_Lst = []
        for url in url_Lst:
            video_id = ' '.join(re.findall('id.*html', url)).split('_')[1
                ].split('.')[0]
            video_id_Lst.append(video_id)

    def get_video_title_play_count_duration(self, url):
        video_id = ' '.join(re.findall('id.*html', url))
        mobile_url = 'https://m.youku.com/video/' + video_id
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        title = browser.find_element_by_class_name('online').text
        dura = browser.find_element_by_class_name('v-label').text.split('\n')[1
            ]
        dura_lst = ' '.join(dura).split[':']
        if len(dura_lst) == 2:
            duration = dura_lst[0] * 60 + dura_lst[1]
        elif len(dura_lst) == 3:
            duration = dura_lst[0] * 3600 + dura_lst[1] * 60 + dura_lst[2]
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',', '')
        pc_midstep4 = re.findall('[0-9]+', pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0] * 10000.0 + pc_midstep4[1] * 1000.0
        else:
            play_count = ' '.join(pc_midstep4)
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now(
            )) * 1000.0)
        D0 = {'video_id': video_id, 'title': title, 'duration': duration,
            'play_count': play_count, 'url': url, 'fetch_time': fetch_time}
        browser.close()
        return D0

    def get_video_play_count_by_vid(self, vid):
        mobile_url = 'https://m.youku.com/video/id_' + vid + '==.html'
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        time.sleep(10)
        title = browser.find_element_by_class_name('online').text
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',', '')
        pc_midstep4 = re.findall('[0-9]+', pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0] * 10000.0 + pc_midstep4[1] * 1000.0
        else:
            play_count = ' '.join(pc_midstep4)
        browser.close()
        D0 = {'title': title, 'play_count': play_count}
        return D0
    <mask token>
    <mask token>
    <mask token>

    def get_releaser_detail(self, url):
        releaser_data_Lst = []
        get_page = requests.get(url)
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('a', {'class': 'username'})['title']
        releaser_intro = soup.find('span', {'class': 'desc'}).text
        total_play_count = soup.find('li', {'class': 'vnum'})['title']
        fans = soup.find('li', {'class': 'snum'})['title']
        midstep = soup.find_all('div', {'class': 'title'})
        for brother in midstep:
            if brother.h3.text == '视频':
                num = brother.find('span').text
                self.TotalVideo_num = num[1:-1]
        D_releaser_page = {'releaser': releaser, 'releaser_intro':
            releaser_intro, 'total_play_count': total_play_count, 'follow':
            fans, 'TotalVideo_num': self.TotalVideo_num}
        releaser_data_Lst.append(D_releaser_page)
        return D_releaser_page

    def get_releaser_info(self, releaserUrl):
        get_page = retry_get_url(releaserUrl)
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('a', {'class': 'username'})['title']
        releaser_id = soup.find('a', {'class': 'user-avatar'})['href']
        D0 = {'releaser': releaser, 'releaser_id': releaser_id}
        return D0
    <mask token>

    def doc_list(self, resultfile, donefile, doneurl, errorfile, area):
        list_data_lst = []
        result = open(resultfile, 'a')
        done = open(donefile, 'a')
        donepage = open(doneurl, 'a')
        error = open(errorfile, 'a')
        count = 1
        video_count = 30
        while video_count == 30 and count <= 30:
            url = ('http://list.youku.com/category/show/c_84_a_' + area +
                '_s_1_d_1_p_' + str(count) + '.html')
            get_page = requests.get(url)
            get_page.encoding = 'utf-8'
            page = get_page.text
            print(url)
            soup = BeautifulSoup(page, 'html.parser')
            midstep = soup.find_all('div', {'class': 'p-thumb'})
            video_count = len(midstep)
            done.write(area)
            done.write(str(count))
            done.write('\n')
            done.flush()
            count += 1
            for line in midstep:
                url_step = line.a['href']
                if url_step[:4] != 'http':
                    url = 'http:' + url_step
                else:
                    url = url_step
                try:
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    soup = BeautifulSoup(page, 'html.parser')
                    print(url)
                    try:
                        donepage.write(url)
                        donepage.write('\n')
                        donepage.flush()
                        album_name = soup.find('h2').a.text
                        album_rating = soup.find('span', {'class': 'score'}
                            ).text
                        sumvideo = soup.find('p', {'class': 'tvintro'}).text
                        sumvideo_int = int(re.findall('\\d+', sumvideo)[0])
                        if sumvideo_int <= 100:
                            midstep = soup.find('div', {'id': 'listitem_page1'}
                                )
                            first_video = midstep.find('div', {'class':
                                'item item-cover current'})
                            url = 'http:' + first_video.a['href']
                            title = first_video['title']
                            dura = first_video.find('span', {'class': 'c-time'}
                                ).text
                            dura_lst = dura.split(':')
                            if len(dura_lst) == 1:
                                duration = int(dura_lst[0])
                            elif len(dura_lst) == 2:
                                duration = int(dura_lst[0]) * 60 + int(dura_lst
                                    [1])
                            elif len(dura_lst) == 3:
                                duration = int(dura_lst[0]) * 3600 + int(
                                    dura_lst[1]) * 60 + int(dura_lst[2])
                            playcount_str = first_video.find('div', {
                                'class': 'status'}).text
                            playcount_str = playcount_str.replace('次播放', '')
                            playcount_str = playcount_str.replace(',', '')
                            thousand_10 = '万'
                            if thousand_10 in playcount_str:
                                playcount_str = playcount_str.replace('万', '')
                                play_count = int(float(playcount_str) * 10000.0
                                    )
                            else:
                                play_count = int(playcount_str)
                            fetch_time = int(datetime.datetime.timestamp(
                                datetime.datetime.now()) * 1000.0)
                            D0 = {'url': url, 'tilte': title, 'duartion':
                                duration, 'play_count': play_count,
                                'album_name': album_name, 'album_rating':
                                album_rating, 'fetch_time': fetch_time}
                            D1 = {'url': url, 'video_infor': D0}
                            list_data_lst.append(D1)
                            json_D1 = json.dumps(D1)
                            result.write(json_D1)
                            result.write('\n')
                            result.flush()
                            print('get one line')
                            other_video = midstep.find_all('div', {'class':
                                'item item-cover'})
                            for line in other_video:
                                url = 'http:' + first_video.a['href']
                                title = first_video['title']
                                dura = first_video.find('span', {'class':
                                    'c-time'}).text
                                dura_lst = dura.split(':')
                                if len(dura_lst) == 1:
                                    duration = int(dura_lst[0])
                                elif len(dura_lst) == 2:
                                    duration = int(dura_lst[0]) * 60 + int(
                                        dura_lst[1])
                                elif len(dura_lst) == 3:
                                    duration = int(dura_lst[0]) * 3600 + int(
                                        dura_lst[1]) * 60 + int(dura_lst[2])
                                playcount_str = first_video.find('div', {
                                    'class': 'status'}).text
                                playcount_str = playcount_str.replace('次播放', ''
                                    )
                                playcount_str = playcount_str.replace(',', '')
                                thousand_10 = '万'
                                if thousand_10 in playcount_str:
                                    playcount_str = playcount_str.replace('万',
                                        '')
                                    play_count = int(float(playcount_str) *
                                        10000.0)
                                else:
                                    play_count = int(playcount_str)
                                fetch_time = int(datetime.datetime.
                                    timestamp(datetime.datetime.now()) * 1000.0
                                    )
                                D0 = {'url': url, 'tilte': title,
                                    'duartion': duration, 'play_count':
                                    play_count, 'album_name': album_name,
                                    'album_rating': album_rating,
                                    'fetch_time': fetch_time}
                                D1 = {'url': url, 'video_infor': D0}
                                list_data_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                print('get one line')
                            result.flush()
                        if sumvideo_int > 100:
                            if sumvideo_int % 50 == 0:
                                page_num = int(sumvideo_int / 50)
                            else:
                                page_num = int(sumvideo_int / 50) + 1
                            videoid = re.findall('\\d+', re.findall(
                                'videoId:"[0-9]{1,10}"', page)[0])[0]
                            showid = re.findall('\\d+', re.findall(
                                'showid:"[0-9]{1,10}"', page)[0])[0]
                            url_lst = [(
                                'http://v.youku.com/page/playlist?&l=debug&pm=3&vid='
                                 + videoid + '&fid=0&showid=' + showid +
                                '&sid=0&page={}&callback=tuijsonp23'.format
                                (str(i))) for i in range(1, page_num + 1)]
                            for line in url_lst:
                                get_page = requests.get(line)
                                get_page.encoding = 'utf-8'
                                page = get_page.text
                                page_dic = eval(page[34:-2])
                                videolab = page_dic['html']
                                videolab = videolab.replace('\\', '')
                                soup = BeautifulSoup(videolab, 'html.parser')
                                first_video = soup.find('div', {'class':
                                    'item item-cover current'})
                                url = 'http:' + first_video.a['href']
                                title = first_video['title']
                                dura = first_video.find('span', {'class':
                                    'c-time'}).text
                                dura_lst = dura.split(':')
                                if len(dura_lst) == 1:
                                    duration = int(dura_lst[0])
                                elif len(dura_lst) == 2:
                                    duration = int(dura_lst[0]) * 60 + int(
                                        dura_lst[1])
                                elif len(dura_lst) == 3:
                                    duration = int(dura_lst[0]) * 3600 + int(
                                        dura_lst[1]) * 60 + int(dura_lst[2])
                                playcount_str = first_video.find('div', {
                                    'class': 'status'}).text
                                playcount_str = playcount_str.replace('次播放', ''
                                    )
                                playcount_str = playcount_str.replace(',', '')
                                thousand_10 = '万'
                                if thousand_10 in playcount_str:
                                    playcount_str = playcount_str.replace('万',
                                        '')
                                    play_count = int(float(playcount_str) *
                                        10000.0)
                                else:
                                    play_count = int(playcount_str)
                                fetch_time = int(datetime.datetime.
                                    timestamp(datetime.datetime.now()) * 1000.0
                                    )
                                D0 = {'url': url, 'tilte': title,
                                    'duartion': duration, 'play_count':
                                    play_count, 'album_name': album_name,
                                    'album_rating': album_rating,
                                    'fetch_time': fetch_time}
                                D1 = {'url': url, 'video_infor': D0}
                                list_data_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print('get one line')
                                other_video = midstep.find_all('div', {
                                    'class': 'item item-cover'})
                                for line in other_video:
                                    url = 'http:' + first_video.a['href']
                                    title = first_video['title']
                                    dura = first_video.find('span', {
                                        'class': 'c-time'}).text
                                    dura_lst = dura.split(':')
                                    if len(dura_lst) == 1:
                                        duration = int(dura_lst[0])
                                    elif len(dura_lst) == 2:
                                        duration = int(dura_lst[0]) * 60 + int(
                                            dura_lst[1])
                                    elif len(dura_lst) == 3:
                                        duration = int(dura_lst[0]) * 3600 + int(
                                            dura_lst[1]) * 60 + int(dura_lst[2])
                                    playcount_str = first_video.find('div',
                                        {'class': 'status'}).text
                                    playcount_str = playcount_str.replace('次播放'
                                        , '')
                                    playcount_str = playcount_str.replace(',',
                                        '')
                                    thousand_10 = '万'
                                    if thousand_10 in playcount_str:
                                        playcount_str = playcount_str.replace('万',
                                            '')
                                        play_count = int(float(playcount_str) *
                                            10000.0)
                                    else:
                                        play_count = int(playcount_str)
                                    fetch_time = int(datetime.datetime.
                                        timestamp(datetime.datetime.now()) *
                                        1000.0)
                                    D0 = {'url': url, 'tilte': title,
                                        'duartion': duration, 'play_count':
                                        play_count, 'album_name':
                                        album_name, 'album_rating':
                                        album_rating, 'fetch_time': fetch_time}
                                    D1 = {'url': url, 'video_infor': D0}
                                    list_data_lst.append(D1)
                                    json_D1 = json.dumps(D1)
                                    result.write(json_D1)
                                    result.write('\n')
                                    print('get one line')
                                result.flush()
                    except:
                        error.write(url)
                        error.write('\n')
                        error.flush()
                        print('there is an error')
                except:
                    error.write(url)
                    error.write('\n')
                    error.flush()
                    print('there is an error')
        done.close()
        result.close()
        return list_data_lst

    def add_infor(self, taskname, resultname):
        task = open(taskname, 'a')
        result = open(resultname, 'a')
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            video_infor = line_dic['video_infor']
            task_lst.append(video_infor)
        for line in task_lst:
            url = line['url']
            try:
                video_dic = self.video_page(url)
                line['releaser'] = video_dic['releaser']
                line['release_time'] = video_dic['release_time']
                line['releaserUrl'] = video_dic['releaserUrl']
                line['tags'] = video_dic['tags']
                json_line = json.dumps(line)
                result_lst.append(line)
                result.write(json_line)
                result.write('\n')
                result.flush()
            except:
                print('no video infor')
        result.close()
        return result_lst

    def get_doc(self, cluename, taskname, resultname, donename):
        task = open(taskname)
        result = open(resultname, 'a')
        clue = open(cluename, 'a')
        done = open(donename, 'a')
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls = line['url']
            urls = urls.replace('\n', '')
            album_name = line['album_name']
            album_rating = line['album_rating']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = ' '.join(re.findall('showid: &#\\d+;\\d+', page)
                    ).split(';')[1]
                videoId = ' '.join(re.findall('videoId: &#\\d+;\\d+', page)
                    ).split(';')[1]
                videoCategoryId = ' '.join(re.findall(
                    'videoCategoryId: &#\\d+;\\d+', page)).split(';')[1]
                componentid = ' '.join(re.findall('"componentId":\\d+', page)
                    ).split(':')[1]
                D0 = {'showid': showid, 'videoid': videoId,
                    'videocategoryid': videoCategoryId, 'componentid':
                    componentid}
                json_D0 = json.dumps(D0)
                clue.write(json_D0)
                clue.write('\n')
                clue.flush()
                count = 1
                thousand_10 = '万'
                url = (
                    'http://v.youku.com/page/playlist?&l=debug&pm=3&vid=' +
                    videoId + '&fid=0&showid=' + showid +
                    '&sid=0&componentid=' + componentid +
                    '&videoCategoryId=' + videoCategoryId +
                    '&isSimple=false&page=' + str(count))
                get_page = requests.get(url)
                get_page.encoding = 'utf-8'
                page = get_page.text
                page_dic = eval(page)
                video_infor = page_dic['html']
                soup = BeautifulSoup(video_infor, 'html.parser')
                video_dic = soup.find_all('div', {'class': 'item item-cover'})
                sumofvideo = len(video_dic)
                for line in video_dic:
                    title = line['title']
                    url = line.a['href']
                    if 'http:' not in url:
                        url = 'http:' + url
                    dura = line.find('span', {'class': 'c-time'}).text
                    dura_lst = dura.split(':')
                    if len(dura_lst) == 1:
                        duration = int(dura_lst[0])
                    elif len(dura_lst) == 2:
                        duration = int(dura_lst[0]) * 60 + int(dura_lst[1])
                    elif len(dura_lst) == 3:
                        duration = int(dura_lst[0]) * 3600 + int(dura_lst[1]
                            ) * 60 + int(dura_lst[2])
                    play_count_str = line.find('div', {'class': 'status'}).text
                    play_count_str = play_count_str.replace('次播放', '')
                    play_count_str = play_count_str.replace(',', '')
                    if thousand_10 in play_count_str:
                        play_count_str = play_count_str.replace('万', '')
                        play_count = int(float(play_count_str) * 10000.0)
                    else:
                        play_count = int(play_count_str)
                    fetch_time = int(datetime.datetime.timestamp(datetime.
                        datetime.now()) * 1000.0)
                    D1 = {'album_name': album_name, 'album_rating':
                        album_rating, 'title': title, 'duration': duration,
                        'play_count': play_count, 'url': url, 'fetch_time':
                        fetch_time}
                    if D1 not in result_lst:
                        result_lst.append(D1)
                        json_D1 = json.dumps(D1)
                        result.write(json_D1)
                        result.write('\n')
                        result.flush()
                        print(url)
                while sumofvideo == 100:
                    count += 1
                    url = (
                        'http://v.youku.com/page/playlist?&l=debug&pm=3&vid=' +
                        videoId + '&fid=0&showid=' + showid +
                        '&sid=0&componentid=' + componentid +
                        '&videoCategoryId=' + videoCategoryId +
                        '&isSimple=false&page=' + str(count))
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    page_dic = eval(page)
                    video_infor = page_dic['html']
                    if video_infor == '\n':
                        sumofvideo = 0
                    else:
                        soup = BeautifulSoup(video_infor, 'html.parser')
                        video_dic = soup.find_all('div', {'class':
                            'item item-cover'})
                        sumofvideo = len(video_dic)
                        for line in video_dic:
                            title = line['title']
                            url = line.a['href']
                            if 'http:' not in url:
                                url = 'http:' + url
                            dura = line.find('span', {'class': 'c-time'}).text
                            dura_lst = dura.split(':')
                            if len(dura_lst) == 1:
                                duration = int(dura_lst[0])
                            elif len(dura_lst) == 2:
                                duration = int(dura_lst[0]) * 60 + int(dura_lst
                                    [1])
                            elif len(dura_lst) == 3:
                                duration = int(dura_lst[0]) * 3600 + int(
                                    dura_lst[1]) * 60 + int(dura_lst[2])
                            play_count_str = line.find('div', {'class':
                                'status'}).text
                            play_count_str = play_count_str.replace('次播放', '')
                            play_count_str = play_count_str.replace(',', '')
                            if thousand_10 in play_count_str:
                                play_count_str = play_count_str.replace('万', ''
                                    )
                                play_count = int(float(play_count_str) * 
                                    10000.0)
                            else:
                                play_count = int(play_count_str)
                            fetch_time = int(datetime.datetime.timestamp(
                                datetime.datetime.now()) * 1000.0)
                            D1 = {'album_name': album_name, 'album_rating':
                                album_rating, 'title': title, 'duration':
                                duration, 'play_count': play_count, 'url':
                                url, 'fetch_time': fetch_time}
                            if D1 not in result_lst:
                                result_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print(url)
                done.write(urls)
                done.write('\n')
                done.flush()
            except:
                pass
        done.close()
        task.close()
        result.close()
        clue.close()
        return result_lst

    def one_video(self, taskname):
        task = open(taskname)
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls = line['url']
            urls = urls.replace('\n', '')
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = ' '.join(re.findall('showid: &#\\d+;\\d+', page)
                    ).split(';')[1]
                print('more video')
            except:
                result_lst.append(urls)
                print('one video')
        task.close()
        return result_lst
    <mask token>
    <mask token>
    <mask token>

    def get_release_time_from_str(self, rt_str):
        minute = '分钟'
        hour = '小时'
        yesterday = '昨天'
        the_day_before_yesterday = '前天'
        the_day_before_many_days = '天前'
        if minute in rt_str or hour in rt_str:
            year = datetime.date.today().year
            month = datetime.date.today().month
            day = datetime.date.today().day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        elif yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days=1)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        elif the_day_before_yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days=2)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        elif the_day_before_many_days in rt_str:
            day_shift = int(re.findall('\\d+', rt_str)[0])
            raw_data = datetime.date.today() - datetime.timedelta(days=
                day_shift)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        else:
            release_time = 0
        return release_time

    def list_page_all(self, output_to_file=False, filepath=None,
        output_to_es_raw=False, output_to_es_register=False, push_to_redis=
        False, page_num_max=25):
        """
        Get all videos in youku.com '片库' in the home page bottom and the '全部'
        category with '最新发布' filter.
        At present (2018-06-28) the url is http://list.youku.com/category/video/c_0_d_1_s_2.html?spm=a2h1n.8251846.selectID.5!2~5~5!2~1~3!2~A
        This url changes to the following format
        'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str(count) + '.html'
        where count is page number.
        """
        if page_num_max > 25:
            page_num_max = 25
        result_Lst = []
        page_num = 1
        while page_num <= page_num_max:
            lst_url = (
                'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str
                (page_num) + '.html')
            print('Getting data on page %d: %s, %s' % (page_num, lst_url,
                datetime.datetime.now()))
            get_page = retry_get_url(lst_url)
            if get_page is None:
                print(
                    'Failed to get page url: %s, will continue to next page.' %
                    lst_url)
                continue
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_agg:
                title = line.find('li', {'class': 'title'}).text
                dura_str = line.find('li', {'class': 'status'}).text
                dura_lst = dura_str.split(':')
                if len(dura_lst) == 3:
                    duration = int(int(dura_lst[0]) * 3600 + int(dura_lst[1
                        ]) * 60 + int(dura_lst[2]))
                elif len(dura_lst) == 2:
                    duration = int(int(dura_lst[0]) * 60 + int(dura_lst[1]))
                else:
                    duration = 0
                url = line.a['href']
                if url[0:4] != 'http':
                    url = 'https:' + url
                video_info_dic = self.get_video_play_count_releaser_info(url)
                fetch_time = int(datetime.datetime.timestamp(datetime.
                    datetime.now()) * 1000.0)
                rt_str = line.find('li', {'class': ' '}).text
                release_time = self.get_release_time_from_str(rt_str)
                self.video_data['title'] = title
                self.video_data['duration'] = duration
                self.video_data['url'] = url
                self.video_data['play_count'] = video_info_dic['play_count']
                self.video_data['releaser'] = video_info_dic['releaser']
                self.video_data['releaserUrl'] = video_info_dic['releaserUrl']
                self.video_data['release_time'] = release_time
                self.video_data['fetch_time'] = fetch_time
                get_data = copy.deepcopy(self.video_data)
                result_Lst.append(get_data)
                if len(result_Lst) >= 100:
                    print('Got %d lines, %s' % (len(result_Lst), datetime.
                        datetime.now()))
                    output_result(result_Lst, self.platform, output_to_file
                        =output_to_file, filepath=filepath,
                        output_to_es_raw=output_to_es_raw,
                        output_to_es_register=output_to_es_register,
                        push_to_redis=push_to_redis)
                    result_Lst.clear()
            page_num += 1
        if result_Lst != []:
            print('Got %d lines, %s' % (len(result_Lst), datetime.datetime.
                now()))
            output_result(result_Lst, self.platform, output_to_file=
                output_to_file, filepath=filepath, output_to_es_raw=
                output_to_es_raw, output_to_es_register=
                output_to_es_register, push_to_redis=push_to_redis)
            result_Lst.clear()

    def get_title_subtitle(self, url):
        get_page = retry_get_url(url)
        if get_page is None:
            print('Failed to get url: %s' % url)
            return
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        try:
            subtitle = soup.find('span', {'id': 'subtitle'}).text
        except:
            subtitle = None
        try:
            title = soup.h1.find('a', {'target': '_blank'}).text
            title_str = title + subtitle
        except:
            title_str = subtitle
        return title_str
    <mask token>

    def start_list_page(self, task_list, output_to_file=False, filepath=
        None, output_to_es_raw=False, push_to_redis=False, es_index=
        'crawler-data-raw', doc_type='doc', output_to_es_register=False):
        self.list_page_task(task_list)
        self.download_list_page_single_process()

    def list_page_task(self, task_list, page_num_max=25):
        TASK_LIST = []
        for line in task_list:
            for num in range(1, page_num_max + 1):
                url = line.replace('fangyucheng', str(num))
                TASK_LIST.append(url)
        random.shuffle(TASK_LIST)
        connect_with_redis.push_list_url_to_redis(platform=self.platform,
            result_lst=TASK_LIST)

    async def download_page(self, session, url):
        get_page = await session.get(url)
        page = await get_page.text('utf-8', errors='ignore')
        return page

    async def download_list_page(self, loop):
        task_list = connect_with_redis.retrieve_list_url_from_redis(platform
            =self.platform)
        async with aiohttp.ClientSession() as session:
            task = [loop.create_task(self.download_page(session, url)) for
                url in task_list]
            done, pending = await asyncio.wait(task)
            RESULT_LIST = [d.result() for d in done]
            connect_with_redis.push_list_page_html_to_redis(platform=self.
                platform, result_lst=RESULT_LIST)
    <mask token>
    <mask token>

    def parse_list_page_multi_process(self, processes_num=20):
        pool = Pool(processes=processes_num)
        for num in range(processes_num):
            pool.apply_async(self.parse_list_page_single_process)
        pool.close()
        pool.join()


<mask token>

<mask token>


class Crawler_youku:

    def __init__(self, timeout=None, platform='youku'):
        if timeout == None:
            self.timeout = 10
        else:
            self.timeout = timeout
        self.platform = platform
        std_fields = Std_fields_video()
        self.video_data = std_fields.video_data
        self.video_data['platform'] = self.platform
        pop_key_Lst = ['channel', 'describe', 'repost_count', 'isOriginal',
            'video_id']
        for popk in pop_key_Lst:
            self.video_data.pop(popk)
        self.legal_channels = ['全部']
        self.list_page_url_dict = {'全部': '_'}

    def video_page(self, url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('span', {'id': 'module_basic_sub'}).a.text
        releaser = releaser.replace('\n', '')
        releaser = releaser.replace(' ', '')
        releaserUrl = soup.find('span', {'id': 'module_basic_sub'}).a['href']
        if releaserUrl[0:4] == 'http':
            pass
        else:
            releaserUrl = 'http:' + releaserUrl
        try:
            rt_str = soup.find('span', {'class': 'video-status'}).text
            rt_lst = re.findall('\\d+', rt_str)
            rt_to_timestamp = str(rt_lst[0]) + '-' + str(rt_lst[1]
                ) + '-' + str(rt_lst[2])
            release_time = int(datetime.datetime.strptime(rt_to_timestamp,
                '%Y-%m-%d').timestamp() * 1000.0)
        except:
            release_time = 0
        D0 = {'releaser': releaser, 'releaserUrl': releaserUrl,
            'release_time': release_time}
        return D0

    def get_video_infor_by_network(self, url_Lst):
        video_id_Lst = []
        for url in url_Lst:
            video_id = ' '.join(re.findall('id.*html', url)).split('_')[1
                ].split('.')[0]
            video_id_Lst.append(video_id)

    def get_video_title_play_count_duration(self, url):
        video_id = ' '.join(re.findall('id.*html', url))
        mobile_url = 'https://m.youku.com/video/' + video_id
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        title = browser.find_element_by_class_name('online').text
        dura = browser.find_element_by_class_name('v-label').text.split('\n')[1
            ]
        dura_lst = ' '.join(dura).split[':']
        if len(dura_lst) == 2:
            duration = dura_lst[0] * 60 + dura_lst[1]
        elif len(dura_lst) == 3:
            duration = dura_lst[0] * 3600 + dura_lst[1] * 60 + dura_lst[2]
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',', '')
        pc_midstep4 = re.findall('[0-9]+', pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0] * 10000.0 + pc_midstep4[1] * 1000.0
        else:
            play_count = ' '.join(pc_midstep4)
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now(
            )) * 1000.0)
        D0 = {'video_id': video_id, 'title': title, 'duration': duration,
            'play_count': play_count, 'url': url, 'fetch_time': fetch_time}
        browser.close()
        return D0

    def get_video_play_count_by_vid(self, vid):
        mobile_url = 'https://m.youku.com/video/id_' + vid + '==.html'
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        time.sleep(10)
        title = browser.find_element_by_class_name('online').text
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',', '')
        pc_midstep4 = re.findall('[0-9]+', pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0] * 10000.0 + pc_midstep4[1] * 1000.0
        else:
            play_count = ' '.join(pc_midstep4)
        browser.close()
        D0 = {'title': title, 'play_count': play_count}
        return D0

    def get_video_title_releaser_release_time(self, url):
        """An example of selenium
        """
        video_id = ' '.join(re.findall('id.*html', url))
        browser = webdriver.Chrome()
        browser.get(url)
        title = browser.find_element_by_id('subtitle').text
        releaser = browser.find_element_by_id('module_basic_sub').text
        releaser = releaser.replace('+订阅', '')
        releaser = releaser.replace(' ', '')
        try:
            rt_midstep = browser.find_element_by_class_name('video-status'
                ).text
            rt_midstep = rt_midstep.replace('上传于', '')
            rt_midstep = rt_midstep.replace(' ', '')
            release_time = int(datetime.datetime.strptime(rt_midstep,
                '%Y-%m-%d').timestamp() * 1000.0)
        except:
            release_time = 0
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now(
            )) * 1000.0)
        D0 = {'video_id': video_id, 'title': title, 'release_time':
            release_time, 'url': url, 'fetch_time': fetch_time}
        return D0
    <mask token>

    def search_page(self, keyword, search_pages_max=10, output_to_es_raw=
        False, output_to_es_register=False, es_index=None, doc_type=None):

        def process_one_line(data_line):
            title = data_line.find('div', {'class': 'v-meta-title'}).a['title']
            url = data_line.find('div', {'class': 'v-meta-title'}).a['href']
            releaser = data_line.find('span', {'class': 'username'}).text
            releaser = releaser.strip()
            releaserUrl = data_line.find('span', {'class': 'username'}).a[
                'href']
            releaserUrl = 'http:' + releaserUrl
            play_count = data_line.find('span', {'class': 'pub'}).text
            dura = data_line.find('span', {'class': 'v-time'}).text
            duration_str = dura
            dl = duration_str.split(':')
            dl_int = []
            for v in dl:
                v = int(v)
                dl_int.append(v)
            if len(dl_int) == 2:
                duration = dl_int[0] * 60 + dl_int[1]
            else:
                duration = dl_int[0] * 3660 + dl_int[1] * 60 + dl_int[2]
            videourl = 'https:' + url
            dict_page = self.video_page(videourl)
            release_time = dict_page['release_time']
            D0 = copy.deepcopy(self.video_data)
            D0['title'] = title
            D0['url'] = videourl
            D0['duration'] = duration
            D0['releaser'] = releaser
            D0['release_time'] = release_time
            D0['releaserUrl'] = releaserUrl
            D0['play_count'] = play_count
            return D0
        search_data_Lst = []
        jsurl = ('http://www.soku.com/search_video_ajax/q_' + keyword +
            '_orderby_1_limitdate_0?site=14&page={}'.format(str(i)) for i in
            range(2, 4))
        for urls in jsurl:
            print(urls)
            get_page = requests.get(urls)
            print('get jspage done')
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            youku = soup.find_all('div', {'class': 'v'})
            for data_line in youku:
                one_video_dic = process_one_line(data_line)
                search_data_Lst.append(one_video_dic)
        url = ('http://www.soku.com/search_video/q_' + keyword +
            '_orderby_1_limitdate_0?spm=a2h0k.8191407.0.0&site=14&page={}'.
            format(str(i)) for i in range(1, search_pages_max + 1))
        for urls in url:
            print(urls)
            get_page = requests.get(urls)
            print('get page done')
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            youku = soup.find_all('div', {'class': 'v'})
            for data_line in youku:
                one_video_dic = process_one_line(data_line)
                search_data_Lst.append(one_video_dic)
                if len(search_data_Lst) >= 100:
                    output_result(result_Lst=search_data_Lst, platform=self
                        .platform, output_to_es_raw=output_to_es_raw,
                        output_to_es_register=output_to_es_register,
                        es_index=es_index, doc_type=doc_type)
                    search_data_Lst.clear()
        if search_data_Lst != []:
            output_result(result_Lst=search_data_Lst, platform=self.
                platform, output_to_es_raw=output_to_es_raw,
                output_to_es_register=output_to_es_register, es_index=
                es_index, doc_type=doc_type)
        return search_data_Lst

    def get_releaser_detail(self, url):
        releaser_data_Lst = []
        get_page = requests.get(url)
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('a', {'class': 'username'})['title']
        releaser_intro = soup.find('span', {'class': 'desc'}).text
        total_play_count = soup.find('li', {'class': 'vnum'})['title']
        fans = soup.find('li', {'class': 'snum'})['title']
        midstep = soup.find_all('div', {'class': 'title'})
        for brother in midstep:
            if brother.h3.text == '视频':
                num = brother.find('span').text
                self.TotalVideo_num = num[1:-1]
        D_releaser_page = {'releaser': releaser, 'releaser_intro':
            releaser_intro, 'total_play_count': total_play_count, 'follow':
            fans, 'TotalVideo_num': self.TotalVideo_num}
        releaser_data_Lst.append(D_releaser_page)
        return D_releaser_page

    def get_releaser_info(self, releaserUrl):
        get_page = retry_get_url(releaserUrl)
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('a', {'class': 'username'})['title']
        releaser_id = soup.find('a', {'class': 'user-avatar'})['href']
        D0 = {'releaser': releaser, 'releaser_id': releaser_id}
        return D0

    @logged
    def releaser_page(self, releaserUrl, error_file=None, output_to_file=
        False, filepath=None, output_to_es_raw=False, output_es_index=None,
        output_doc_type=None, output_to_es_register=False, push_to_redis=
        False, releaser_page_num_max=300):
        result_Lst = []
        releaser_info = self.get_releaser_info(releaserUrl)
        releaser = releaser_info['releaser']
        releaser_id = releaser_info['releaser_id']
        count = 1
        while count <= releaser_page_num_max:
            releaser_url = ('https://i.youku.com' + releaser_id +
                '/videos?&order=1&page=' + str(count))
            count += 1
            get_page = retry_get_url(releaser_url)
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_lst = soup.find_all('div', {'class': 'v va'})
            for one_video in video_lst:
                try:
                    title = one_video.find('div', {'class': 'v-meta-title'}).a[
                        'title']
                    url = one_video.find('div', {'class': 'v-meta-title'}).a[
                        'href']
                    if url[0:4] != 'http':
                        url = 'https:' + url
                    try:
                        play_count_text = one_video.find('span', {'class':
                            'v-num'}).text
                        if '万' in play_count_text:
                            play_count_str = play_count_text.replace('万', '')
                            play_count = int(float(play_count_str) * 10000.0)
                        else:
                            play_count = int(play_count_text.replace(',', ''))
                    except:
                        play_count = 0
                    dura_str = one_video.find('span', {'class': 'v-time'}).text
                    dura_lst = dura_str.split(':')
                    if len(dura_lst) == 2:
                        duration = int(dura_lst[0]) * 60 + int(dura_lst[1])
                    elif len(dura_lst) == 3:
                        duration = int(dura_lst[0]) * 3600 + int(dura_lst[1]
                            ) * 60 + int(dura_lst[2])
                    else:
                        duration = 0
                    rt_str = one_video.find('span', {'class': 'v-publishtime'}
                        ).text
                    release_time = (trans_strtime_to_timestamp.
                        trans_strtime_to_timestamp(rt_str, missing_year=True))
                    fetch_time = int(datetime.datetime.timestamp(datetime.
                        datetime.now()) * 1000.0)
                    self.video_data['title'] = title
                    self.video_data['duration'] = duration
                    self.video_data['url'] = url
                    self.video_data['play_count'] = play_count
                    self.video_data['releaser'] = releaser
                    self.video_data['releaserUrl'] = releaserUrl
                    self.video_data['release_time'] = release_time
                    self.video_data['fetch_time'] = fetch_time
                    get_data = copy.deepcopy(self.video_data)
                    result_Lst.append(get_data)
                    print('get_one_line %s' % url)
                except:
                    print('failed to get data')
                    pass
                if len(result_Lst) % 100 == 0:
                    if output_doc_type is None and output_es_index is None:
                        output_result(result_Lst, self.platform,
                            output_to_file=output_to_file, filepath=
                            filepath, output_to_es_raw=output_to_es_raw,
                            output_to_es_register=output_to_es_register,
                            push_to_redis=push_to_redis)
                        result_Lst.clear()
                    elif output_doc_type is not None and output_es_index is not None:
                        output_result(result_Lst, self.platform,
                            output_to_file=output_to_file, filepath=
                            filepath, output_to_es_raw=output_to_es_raw,
                            output_to_es_register=output_to_es_register,
                            push_to_redis=push_to_redis, es_index=
                            output_es_index, doc_type=output_doc_type)
                        result_Lst.clear()
        if result_Lst != []:
            if output_doc_type is None and output_es_index is None:
                output_result(result_Lst, self.platform, output_to_file=
                    output_to_file, filepath=filepath, output_to_es_raw=
                    output_to_es_raw, output_to_es_register=
                    output_to_es_register, push_to_redis=push_to_redis)
            elif output_doc_type is not None and output_es_index is not None:
                output_result(result_Lst, self.platform, output_to_file=
                    output_to_file, filepath=filepath, output_to_es_raw=
                    output_to_es_raw, output_to_es_register=
                    output_to_es_register, push_to_redis=push_to_redis,
                    es_index=output_es_index, doc_type=output_doc_type)

    def doc_list(self, resultfile, donefile, doneurl, errorfile, area):
        list_data_lst = []
        result = open(resultfile, 'a')
        done = open(donefile, 'a')
        donepage = open(doneurl, 'a')
        error = open(errorfile, 'a')
        count = 1
        video_count = 30
        while video_count == 30 and count <= 30:
            url = ('http://list.youku.com/category/show/c_84_a_' + area +
                '_s_1_d_1_p_' + str(count) + '.html')
            get_page = requests.get(url)
            get_page.encoding = 'utf-8'
            page = get_page.text
            print(url)
            soup = BeautifulSoup(page, 'html.parser')
            midstep = soup.find_all('div', {'class': 'p-thumb'})
            video_count = len(midstep)
            done.write(area)
            done.write(str(count))
            done.write('\n')
            done.flush()
            count += 1
            for line in midstep:
                url_step = line.a['href']
                if url_step[:4] != 'http':
                    url = 'http:' + url_step
                else:
                    url = url_step
                try:
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    soup = BeautifulSoup(page, 'html.parser')
                    print(url)
                    try:
                        donepage.write(url)
                        donepage.write('\n')
                        donepage.flush()
                        album_name = soup.find('h2').a.text
                        album_rating = soup.find('span', {'class': 'score'}
                            ).text
                        sumvideo = soup.find('p', {'class': 'tvintro'}).text
                        sumvideo_int = int(re.findall('\\d+', sumvideo)[0])
                        if sumvideo_int <= 100:
                            midstep = soup.find('div', {'id': 'listitem_page1'}
                                )
                            first_video = midstep.find('div', {'class':
                                'item item-cover current'})
                            url = 'http:' + first_video.a['href']
                            title = first_video['title']
                            dura = first_video.find('span', {'class': 'c-time'}
                                ).text
                            dura_lst = dura.split(':')
                            if len(dura_lst) == 1:
                                duration = int(dura_lst[0])
                            elif len(dura_lst) == 2:
                                duration = int(dura_lst[0]) * 60 + int(dura_lst
                                    [1])
                            elif len(dura_lst) == 3:
                                duration = int(dura_lst[0]) * 3600 + int(
                                    dura_lst[1]) * 60 + int(dura_lst[2])
                            playcount_str = first_video.find('div', {
                                'class': 'status'}).text
                            playcount_str = playcount_str.replace('次播放', '')
                            playcount_str = playcount_str.replace(',', '')
                            thousand_10 = '万'
                            if thousand_10 in playcount_str:
                                playcount_str = playcount_str.replace('万', '')
                                play_count = int(float(playcount_str) * 10000.0
                                    )
                            else:
                                play_count = int(playcount_str)
                            fetch_time = int(datetime.datetime.timestamp(
                                datetime.datetime.now()) * 1000.0)
                            D0 = {'url': url, 'tilte': title, 'duartion':
                                duration, 'play_count': play_count,
                                'album_name': album_name, 'album_rating':
                                album_rating, 'fetch_time': fetch_time}
                            D1 = {'url': url, 'video_infor': D0}
                            list_data_lst.append(D1)
                            json_D1 = json.dumps(D1)
                            result.write(json_D1)
                            result.write('\n')
                            result.flush()
                            print('get one line')
                            other_video = midstep.find_all('div', {'class':
                                'item item-cover'})
                            for line in other_video:
                                url = 'http:' + first_video.a['href']
                                title = first_video['title']
                                dura = first_video.find('span', {'class':
                                    'c-time'}).text
                                dura_lst = dura.split(':')
                                if len(dura_lst) == 1:
                                    duration = int(dura_lst[0])
                                elif len(dura_lst) == 2:
                                    duration = int(dura_lst[0]) * 60 + int(
                                        dura_lst[1])
                                elif len(dura_lst) == 3:
                                    duration = int(dura_lst[0]) * 3600 + int(
                                        dura_lst[1]) * 60 + int(dura_lst[2])
                                playcount_str = first_video.find('div', {
                                    'class': 'status'}).text
                                playcount_str = playcount_str.replace('次播放', ''
                                    )
                                playcount_str = playcount_str.replace(',', '')
                                thousand_10 = '万'
                                if thousand_10 in playcount_str:
                                    playcount_str = playcount_str.replace('万',
                                        '')
                                    play_count = int(float(playcount_str) *
                                        10000.0)
                                else:
                                    play_count = int(playcount_str)
                                fetch_time = int(datetime.datetime.
                                    timestamp(datetime.datetime.now()) * 1000.0
                                    )
                                D0 = {'url': url, 'tilte': title,
                                    'duartion': duration, 'play_count':
                                    play_count, 'album_name': album_name,
                                    'album_rating': album_rating,
                                    'fetch_time': fetch_time}
                                D1 = {'url': url, 'video_infor': D0}
                                list_data_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                print('get one line')
                            result.flush()
                        if sumvideo_int > 100:
                            if sumvideo_int % 50 == 0:
                                page_num = int(sumvideo_int / 50)
                            else:
                                page_num = int(sumvideo_int / 50) + 1
                            videoid = re.findall('\\d+', re.findall(
                                'videoId:"[0-9]{1,10}"', page)[0])[0]
                            showid = re.findall('\\d+', re.findall(
                                'showid:"[0-9]{1,10}"', page)[0])[0]
                            url_lst = [(
                                'http://v.youku.com/page/playlist?&l=debug&pm=3&vid='
                                 + videoid + '&fid=0&showid=' + showid +
                                '&sid=0&page={}&callback=tuijsonp23'.format
                                (str(i))) for i in range(1, page_num + 1)]
                            for line in url_lst:
                                get_page = requests.get(line)
                                get_page.encoding = 'utf-8'
                                page = get_page.text
                                page_dic = eval(page[34:-2])
                                videolab = page_dic['html']
                                videolab = videolab.replace('\\', '')
                                soup = BeautifulSoup(videolab, 'html.parser')
                                first_video = soup.find('div', {'class':
                                    'item item-cover current'})
                                url = 'http:' + first_video.a['href']
                                title = first_video['title']
                                dura = first_video.find('span', {'class':
                                    'c-time'}).text
                                dura_lst = dura.split(':')
                                if len(dura_lst) == 1:
                                    duration = int(dura_lst[0])
                                elif len(dura_lst) == 2:
                                    duration = int(dura_lst[0]) * 60 + int(
                                        dura_lst[1])
                                elif len(dura_lst) == 3:
                                    duration = int(dura_lst[0]) * 3600 + int(
                                        dura_lst[1]) * 60 + int(dura_lst[2])
                                playcount_str = first_video.find('div', {
                                    'class': 'status'}).text
                                playcount_str = playcount_str.replace('次播放', ''
                                    )
                                playcount_str = playcount_str.replace(',', '')
                                thousand_10 = '万'
                                if thousand_10 in playcount_str:
                                    playcount_str = playcount_str.replace('万',
                                        '')
                                    play_count = int(float(playcount_str) *
                                        10000.0)
                                else:
                                    play_count = int(playcount_str)
                                fetch_time = int(datetime.datetime.
                                    timestamp(datetime.datetime.now()) * 1000.0
                                    )
                                D0 = {'url': url, 'tilte': title,
                                    'duartion': duration, 'play_count':
                                    play_count, 'album_name': album_name,
                                    'album_rating': album_rating,
                                    'fetch_time': fetch_time}
                                D1 = {'url': url, 'video_infor': D0}
                                list_data_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print('get one line')
                                other_video = midstep.find_all('div', {
                                    'class': 'item item-cover'})
                                for line in other_video:
                                    url = 'http:' + first_video.a['href']
                                    title = first_video['title']
                                    dura = first_video.find('span', {
                                        'class': 'c-time'}).text
                                    dura_lst = dura.split(':')
                                    if len(dura_lst) == 1:
                                        duration = int(dura_lst[0])
                                    elif len(dura_lst) == 2:
                                        duration = int(dura_lst[0]) * 60 + int(
                                            dura_lst[1])
                                    elif len(dura_lst) == 3:
                                        duration = int(dura_lst[0]) * 3600 + int(
                                            dura_lst[1]) * 60 + int(dura_lst[2])
                                    playcount_str = first_video.find('div',
                                        {'class': 'status'}).text
                                    playcount_str = playcount_str.replace('次播放'
                                        , '')
                                    playcount_str = playcount_str.replace(',',
                                        '')
                                    thousand_10 = '万'
                                    if thousand_10 in playcount_str:
                                        playcount_str = playcount_str.replace('万',
                                            '')
                                        play_count = int(float(playcount_str) *
                                            10000.0)
                                    else:
                                        play_count = int(playcount_str)
                                    fetch_time = int(datetime.datetime.
                                        timestamp(datetime.datetime.now()) *
                                        1000.0)
                                    D0 = {'url': url, 'tilte': title,
                                        'duartion': duration, 'play_count':
                                        play_count, 'album_name':
                                        album_name, 'album_rating':
                                        album_rating, 'fetch_time': fetch_time}
                                    D1 = {'url': url, 'video_infor': D0}
                                    list_data_lst.append(D1)
                                    json_D1 = json.dumps(D1)
                                    result.write(json_D1)
                                    result.write('\n')
                                    print('get one line')
                                result.flush()
                    except:
                        error.write(url)
                        error.write('\n')
                        error.flush()
                        print('there is an error')
                except:
                    error.write(url)
                    error.write('\n')
                    error.flush()
                    print('there is an error')
        done.close()
        result.close()
        return list_data_lst

    def add_infor(self, taskname, resultname):
        task = open(taskname, 'a')
        result = open(resultname, 'a')
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            video_infor = line_dic['video_infor']
            task_lst.append(video_infor)
        for line in task_lst:
            url = line['url']
            try:
                video_dic = self.video_page(url)
                line['releaser'] = video_dic['releaser']
                line['release_time'] = video_dic['release_time']
                line['releaserUrl'] = video_dic['releaserUrl']
                line['tags'] = video_dic['tags']
                json_line = json.dumps(line)
                result_lst.append(line)
                result.write(json_line)
                result.write('\n')
                result.flush()
            except:
                print('no video infor')
        result.close()
        return result_lst

    def get_doc(self, cluename, taskname, resultname, donename):
        task = open(taskname)
        result = open(resultname, 'a')
        clue = open(cluename, 'a')
        done = open(donename, 'a')
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls = line['url']
            urls = urls.replace('\n', '')
            album_name = line['album_name']
            album_rating = line['album_rating']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = ' '.join(re.findall('showid: &#\\d+;\\d+', page)
                    ).split(';')[1]
                videoId = ' '.join(re.findall('videoId: &#\\d+;\\d+', page)
                    ).split(';')[1]
                videoCategoryId = ' '.join(re.findall(
                    'videoCategoryId: &#\\d+;\\d+', page)).split(';')[1]
                componentid = ' '.join(re.findall('"componentId":\\d+', page)
                    ).split(':')[1]
                D0 = {'showid': showid, 'videoid': videoId,
                    'videocategoryid': videoCategoryId, 'componentid':
                    componentid}
                json_D0 = json.dumps(D0)
                clue.write(json_D0)
                clue.write('\n')
                clue.flush()
                count = 1
                thousand_10 = '万'
                url = (
                    'http://v.youku.com/page/playlist?&l=debug&pm=3&vid=' +
                    videoId + '&fid=0&showid=' + showid +
                    '&sid=0&componentid=' + componentid +
                    '&videoCategoryId=' + videoCategoryId +
                    '&isSimple=false&page=' + str(count))
                get_page = requests.get(url)
                get_page.encoding = 'utf-8'
                page = get_page.text
                page_dic = eval(page)
                video_infor = page_dic['html']
                soup = BeautifulSoup(video_infor, 'html.parser')
                video_dic = soup.find_all('div', {'class': 'item item-cover'})
                sumofvideo = len(video_dic)
                for line in video_dic:
                    title = line['title']
                    url = line.a['href']
                    if 'http:' not in url:
                        url = 'http:' + url
                    dura = line.find('span', {'class': 'c-time'}).text
                    dura_lst = dura.split(':')
                    if len(dura_lst) == 1:
                        duration = int(dura_lst[0])
                    elif len(dura_lst) == 2:
                        duration = int(dura_lst[0]) * 60 + int(dura_lst[1])
                    elif len(dura_lst) == 3:
                        duration = int(dura_lst[0]) * 3600 + int(dura_lst[1]
                            ) * 60 + int(dura_lst[2])
                    play_count_str = line.find('div', {'class': 'status'}).text
                    play_count_str = play_count_str.replace('次播放', '')
                    play_count_str = play_count_str.replace(',', '')
                    if thousand_10 in play_count_str:
                        play_count_str = play_count_str.replace('万', '')
                        play_count = int(float(play_count_str) * 10000.0)
                    else:
                        play_count = int(play_count_str)
                    fetch_time = int(datetime.datetime.timestamp(datetime.
                        datetime.now()) * 1000.0)
                    D1 = {'album_name': album_name, 'album_rating':
                        album_rating, 'title': title, 'duration': duration,
                        'play_count': play_count, 'url': url, 'fetch_time':
                        fetch_time}
                    if D1 not in result_lst:
                        result_lst.append(D1)
                        json_D1 = json.dumps(D1)
                        result.write(json_D1)
                        result.write('\n')
                        result.flush()
                        print(url)
                while sumofvideo == 100:
                    count += 1
                    url = (
                        'http://v.youku.com/page/playlist?&l=debug&pm=3&vid=' +
                        videoId + '&fid=0&showid=' + showid +
                        '&sid=0&componentid=' + componentid +
                        '&videoCategoryId=' + videoCategoryId +
                        '&isSimple=false&page=' + str(count))
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    page_dic = eval(page)
                    video_infor = page_dic['html']
                    if video_infor == '\n':
                        sumofvideo = 0
                    else:
                        soup = BeautifulSoup(video_infor, 'html.parser')
                        video_dic = soup.find_all('div', {'class':
                            'item item-cover'})
                        sumofvideo = len(video_dic)
                        for line in video_dic:
                            title = line['title']
                            url = line.a['href']
                            if 'http:' not in url:
                                url = 'http:' + url
                            dura = line.find('span', {'class': 'c-time'}).text
                            dura_lst = dura.split(':')
                            if len(dura_lst) == 1:
                                duration = int(dura_lst[0])
                            elif len(dura_lst) == 2:
                                duration = int(dura_lst[0]) * 60 + int(dura_lst
                                    [1])
                            elif len(dura_lst) == 3:
                                duration = int(dura_lst[0]) * 3600 + int(
                                    dura_lst[1]) * 60 + int(dura_lst[2])
                            play_count_str = line.find('div', {'class':
                                'status'}).text
                            play_count_str = play_count_str.replace('次播放', '')
                            play_count_str = play_count_str.replace(',', '')
                            if thousand_10 in play_count_str:
                                play_count_str = play_count_str.replace('万', ''
                                    )
                                play_count = int(float(play_count_str) * 
                                    10000.0)
                            else:
                                play_count = int(play_count_str)
                            fetch_time = int(datetime.datetime.timestamp(
                                datetime.datetime.now()) * 1000.0)
                            D1 = {'album_name': album_name, 'album_rating':
                                album_rating, 'title': title, 'duration':
                                duration, 'play_count': play_count, 'url':
                                url, 'fetch_time': fetch_time}
                            if D1 not in result_lst:
                                result_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print(url)
                done.write(urls)
                done.write('\n')
                done.flush()
            except:
                pass
        done.close()
        task.close()
        result.close()
        clue.close()
        return result_lst

    def one_video(self, taskname):
        task = open(taskname)
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls = line['url']
            urls = urls.replace('\n', '')
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = ' '.join(re.findall('showid: &#\\d+;\\d+', page)
                    ).split(';')[1]
                print('more video')
            except:
                result_lst.append(urls)
                print('one video')
        task.close()
        return result_lst

    def get_releaser(self, taskname, resultname, errorname):
        task = open(taskname)
        result = open(resultname, 'a')
        error = open(errorname, 'a')
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls = line['url']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            page2 = page.replace('\\', '')
            try:
                releaser = re.findall('[一-龥]+', ''.join(re.findall(
                    'class="" title="[一-龥]+"', page2)))[0]
                print('get releaser')
            except:
                releaser = None
                error.write(urls)
                error.write('\n')
                error.flush()
                print('no releaser')
            line['releaser'] = releaser
            result_lst.append(line)
            json_line = json.dumps(line)
            result.write(json_line)
            result.write('\n')
            result.flush()
        result.close()
        return result_lst
    <mask token>

    def get_video_play_count_releaser_info(self, url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        try:
            video_info = soup.find('div', {'class': 'item item-cover current'})
            play_count_str = video_info.find('div', {'class': 'status'}).text
            play_count_str = play_count_str.replace('次播放', '')
            if '万' in play_count_str:
                play_count_str = play_count_str.replace('万', '')
                play_count = int(float(play_count_str) * 10000.0)
            else:
                play_count = int(play_count_str.replace(',', ''))
        except:
            play_count = 0
        try:
            releaser = soup.find('span', {'id': 'module_basic_sub'}).a.text
            releaser = releaser.replace('\n', '')
            releaser = releaser.replace(' ', '')
            releaserUrl = soup.find('span', {'id': 'module_basic_sub'}).a[
                'href']
            if releaserUrl[0:4] == 'http':
                pass
            else:
                releaserUrl = 'http:' + releaserUrl
        except:
            releaser = None
            releaserUrl = None
        D0 = {'play_count': play_count, 'releaser': releaser, 'releaserUrl':
            releaserUrl}
        return D0

    def get_release_time_from_str(self, rt_str):
        minute = '分钟'
        hour = '小时'
        yesterday = '昨天'
        the_day_before_yesterday = '前天'
        the_day_before_many_days = '天前'
        if minute in rt_str or hour in rt_str:
            year = datetime.date.today().year
            month = datetime.date.today().month
            day = datetime.date.today().day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        elif yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days=1)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        elif the_day_before_yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days=2)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        elif the_day_before_many_days in rt_str:
            day_shift = int(re.findall('\\d+', rt_str)[0])
            raw_data = datetime.date.today() - datetime.timedelta(days=
                day_shift)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        else:
            release_time = 0
        return release_time

    def list_page_all(self, output_to_file=False, filepath=None,
        output_to_es_raw=False, output_to_es_register=False, push_to_redis=
        False, page_num_max=25):
        """
        Get all videos in youku.com '片库' in the home page bottom and the '全部'
        category with '最新发布' filter.
        At present (2018-06-28) the url is http://list.youku.com/category/video/c_0_d_1_s_2.html?spm=a2h1n.8251846.selectID.5!2~5~5!2~1~3!2~A
        This url changes to the following format
        'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str(count) + '.html'
        where count is page number.
        """
        if page_num_max > 25:
            page_num_max = 25
        result_Lst = []
        page_num = 1
        while page_num <= page_num_max:
            lst_url = (
                'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str
                (page_num) + '.html')
            print('Getting data on page %d: %s, %s' % (page_num, lst_url,
                datetime.datetime.now()))
            get_page = retry_get_url(lst_url)
            if get_page is None:
                print(
                    'Failed to get page url: %s, will continue to next page.' %
                    lst_url)
                continue
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_agg:
                title = line.find('li', {'class': 'title'}).text
                dura_str = line.find('li', {'class': 'status'}).text
                dura_lst = dura_str.split(':')
                if len(dura_lst) == 3:
                    duration = int(int(dura_lst[0]) * 3600 + int(dura_lst[1
                        ]) * 60 + int(dura_lst[2]))
                elif len(dura_lst) == 2:
                    duration = int(int(dura_lst[0]) * 60 + int(dura_lst[1]))
                else:
                    duration = 0
                url = line.a['href']
                if url[0:4] != 'http':
                    url = 'https:' + url
                video_info_dic = self.get_video_play_count_releaser_info(url)
                fetch_time = int(datetime.datetime.timestamp(datetime.
                    datetime.now()) * 1000.0)
                rt_str = line.find('li', {'class': ' '}).text
                release_time = self.get_release_time_from_str(rt_str)
                self.video_data['title'] = title
                self.video_data['duration'] = duration
                self.video_data['url'] = url
                self.video_data['play_count'] = video_info_dic['play_count']
                self.video_data['releaser'] = video_info_dic['releaser']
                self.video_data['releaserUrl'] = video_info_dic['releaserUrl']
                self.video_data['release_time'] = release_time
                self.video_data['fetch_time'] = fetch_time
                get_data = copy.deepcopy(self.video_data)
                result_Lst.append(get_data)
                if len(result_Lst) >= 100:
                    print('Got %d lines, %s' % (len(result_Lst), datetime.
                        datetime.now()))
                    output_result(result_Lst, self.platform, output_to_file
                        =output_to_file, filepath=filepath,
                        output_to_es_raw=output_to_es_raw,
                        output_to_es_register=output_to_es_register,
                        push_to_redis=push_to_redis)
                    result_Lst.clear()
            page_num += 1
        if result_Lst != []:
            print('Got %d lines, %s' % (len(result_Lst), datetime.datetime.
                now()))
            output_result(result_Lst, self.platform, output_to_file=
                output_to_file, filepath=filepath, output_to_es_raw=
                output_to_es_raw, output_to_es_register=
                output_to_es_register, push_to_redis=push_to_redis)
            result_Lst.clear()

    def get_title_subtitle(self, url):
        get_page = retry_get_url(url)
        if get_page is None:
            print('Failed to get url: %s' % url)
            return
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        try:
            subtitle = soup.find('span', {'id': 'subtitle'}).text
        except:
            subtitle = None
        try:
            title = soup.h1.find('a', {'target': '_blank'}).text
            title_str = title + subtitle
        except:
            title_str = subtitle
        return title_str

    def select_tv_series_video(self, sp_video_album_name, result_file=None,
        output_to_file=False, filepath=None, output_to_es_raw=False,
        output_to_es_register=False, push_to_redis=False):
        select_result = []
        meta = Meta()
        result_lst = meta.dic_file_to_lst(result_file)
        for line in result_lst:
            url = line['url']
            title_str = self.get_title_subtitle(url)
            if sp_video_album_name in title_str:
                line.update({'title': title_str})
                select_result.append(line)
                print('it_belongs_to_what_we_want')
                if len(select_result) >= 100:
                    print('Got %d lines, %s' % (len(select_result),
                        datetime.datetime.now()))
                    output_result(select_result, self.platform,
                        output_to_file=output_to_file, filepath=filepath,
                        output_to_es_raw=output_to_es_raw,
                        output_to_es_register=output_to_es_register,
                        push_to_redis=push_to_redis)
                    select_result.clear()
            else:
                print('not_belongs_to')
        if select_result != []:
            print('Got %d lines, %s' % (len(select_result), datetime.
                datetime.now()))
            output_result(select_result, self.platform, output_to_file=
                output_to_file, filepath=filepath, output_to_es_raw=
                output_to_es_raw, output_to_es_register=
                output_to_es_register, push_to_redis=push_to_redis)
            select_result.clear()

    def start_list_page(self, task_list, output_to_file=False, filepath=
        None, output_to_es_raw=False, push_to_redis=False, es_index=
        'crawler-data-raw', doc_type='doc', output_to_es_register=False):
        self.list_page_task(task_list)
        self.download_list_page_single_process()

    def list_page_task(self, task_list, page_num_max=25):
        TASK_LIST = []
        for line in task_list:
            for num in range(1, page_num_max + 1):
                url = line.replace('fangyucheng', str(num))
                TASK_LIST.append(url)
        random.shuffle(TASK_LIST)
        connect_with_redis.push_list_url_to_redis(platform=self.platform,
            result_lst=TASK_LIST)

    async def download_page(self, session, url):
        get_page = await session.get(url)
        page = await get_page.text('utf-8', errors='ignore')
        return page

    async def download_list_page(self, loop):
        task_list = connect_with_redis.retrieve_list_url_from_redis(platform
            =self.platform)
        async with aiohttp.ClientSession() as session:
            task = [loop.create_task(self.download_page(session, url)) for
                url in task_list]
            done, pending = await asyncio.wait(task)
            RESULT_LIST = [d.result() for d in done]
            connect_with_redis.push_list_page_html_to_redis(platform=self.
                platform, result_lst=RESULT_LIST)

    def download_list_page_single_process(self):
        key = 'youku_list_url'
        while connect_with_redis.length_of_lst(key) > 0:
            loop = asyncio.get_event_loop()
            loop.run_until_complete(self.download_list_page(loop))
    <mask token>

    def parse_list_page_multi_process(self, processes_num=20):
        pool = Pool(processes=processes_num)
        for num in range(processes_num):
            pool.apply_async(self.parse_list_page_single_process)
        pool.close()
        pool.join()


<mask token>

<mask token>
import os
import time
import aiohttp
import asyncio
import requests
from bs4 import BeautifulSoup
import datetime
import re
import json
import copy
import random
from multiprocessing import Pool
from multiprocessing import Process
from crawler_sys.framework.video_fields_std import Std_fields_video
from crawler_sys.utils.output_results import retry_get_url
from crawler_sys.utils.output_results import output_result
from crawler_sys.utils import trans_strtime_to_timestamp
from crawler.crawler_sys.utils import connect_with_redis
from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
from crawler.crawler_sys.utils.util_logging import logged


class Crawler_youku:

    def __init__(self, timeout=None, platform='youku'):
        if timeout == None:
            self.timeout = 10
        else:
            self.timeout = timeout
        self.platform = platform
        std_fields = Std_fields_video()
        self.video_data = std_fields.video_data
        self.video_data['platform'] = self.platform
        pop_key_Lst = ['channel', 'describe', 'repost_count', 'isOriginal',
            'video_id']
        for popk in pop_key_Lst:
            self.video_data.pop(popk)
        self.legal_channels = ['全部']
        self.list_page_url_dict = {'全部': '_'}

    def video_page(self, url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('span', {'id': 'module_basic_sub'}).a.text
        releaser = releaser.replace('\n', '')
        releaser = releaser.replace(' ', '')
        releaserUrl = soup.find('span', {'id': 'module_basic_sub'}).a['href']
        if releaserUrl[0:4] == 'http':
            pass
        else:
            releaserUrl = 'http:' + releaserUrl
        try:
            rt_str = soup.find('span', {'class': 'video-status'}).text
            rt_lst = re.findall('\\d+', rt_str)
            rt_to_timestamp = str(rt_lst[0]) + '-' + str(rt_lst[1]
                ) + '-' + str(rt_lst[2])
            release_time = int(datetime.datetime.strptime(rt_to_timestamp,
                '%Y-%m-%d').timestamp() * 1000.0)
        except:
            release_time = 0
        D0 = {'releaser': releaser, 'releaserUrl': releaserUrl,
            'release_time': release_time}
        return D0

    def get_video_infor_by_network(self, url_Lst):
        video_id_Lst = []
        for url in url_Lst:
            video_id = ' '.join(re.findall('id.*html', url)).split('_')[1
                ].split('.')[0]
            video_id_Lst.append(video_id)

    def get_video_title_play_count_duration(self, url):
        video_id = ' '.join(re.findall('id.*html', url))
        mobile_url = 'https://m.youku.com/video/' + video_id
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        title = browser.find_element_by_class_name('online').text
        dura = browser.find_element_by_class_name('v-label').text.split('\n')[1
            ]
        dura_lst = ' '.join(dura).split[':']
        if len(dura_lst) == 2:
            duration = dura_lst[0] * 60 + dura_lst[1]
        elif len(dura_lst) == 3:
            duration = dura_lst[0] * 3600 + dura_lst[1] * 60 + dura_lst[2]
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',', '')
        pc_midstep4 = re.findall('[0-9]+', pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0] * 10000.0 + pc_midstep4[1] * 1000.0
        else:
            play_count = ' '.join(pc_midstep4)
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now(
            )) * 1000.0)
        D0 = {'video_id': video_id, 'title': title, 'duration': duration,
            'play_count': play_count, 'url': url, 'fetch_time': fetch_time}
        browser.close()
        return D0

    def get_video_play_count_by_vid(self, vid):
        mobile_url = 'https://m.youku.com/video/id_' + vid + '==.html'
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        time.sleep(10)
        title = browser.find_element_by_class_name('online').text
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',', '')
        pc_midstep4 = re.findall('[0-9]+', pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0] * 10000.0 + pc_midstep4[1] * 1000.0
        else:
            play_count = ' '.join(pc_midstep4)
        browser.close()
        D0 = {'title': title, 'play_count': play_count}
        return D0

    def get_video_title_releaser_release_time(self, url):
        """An example of selenium
        """
        video_id = ' '.join(re.findall('id.*html', url))
        browser = webdriver.Chrome()
        browser.get(url)
        title = browser.find_element_by_id('subtitle').text
        releaser = browser.find_element_by_id('module_basic_sub').text
        releaser = releaser.replace('+订阅', '')
        releaser = releaser.replace(' ', '')
        try:
            rt_midstep = browser.find_element_by_class_name('video-status'
                ).text
            rt_midstep = rt_midstep.replace('上传于', '')
            rt_midstep = rt_midstep.replace(' ', '')
            release_time = int(datetime.datetime.strptime(rt_midstep,
                '%Y-%m-%d').timestamp() * 1000.0)
        except:
            release_time = 0
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now(
            )) * 1000.0)
        D0 = {'video_id': video_id, 'title': title, 'release_time':
            release_time, 'url': url, 'fetch_time': fetch_time}
        return D0

    def list_page_sync(self, list_page_url, channel=None, output_to_file=
        False, filepath=None, output_to_es_raw=False, output_to_es_register
        =False, push_to_redis=False, page_num_max=30):
        if channel == '全部':
            self.list_page_all(output_to_file=output_to_file, filepath=
                filepath, output_to_es_raw=output_to_es_raw,
                output_to_es_register=output_to_es_register, push_to_redis=
                push_to_redis, page_num_max=page_num_max)
        else:
            get_page = requests.get(list_page_url)
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video = soup.find_all('div', {'class': 'v-link'})
            dict_url = []
            for ttt in video:
                midstepurl = ttt.find('a')['href']
                if 'https:' in midstepurl:
                    midstepurl = midstepurl[6:]
                url = 'http:' + midstepurl
                dict_url.append(url)
            for xxxx in dict_url:
                if 'id' not in xxxx:
                    num = dict_url.index(xxxx)
                    dict_url.pop(num)
            for xxxx in dict_url:
                if 'id' not in xxxx:
                    num = dict_url.index(xxxx)
                    dict_url.pop(num)
            list_page_Lst = []
            for videourl in dict_url:
                one_video_dic = self.video_page(videourl)
                print('one page done')
                list_page_Lst.append(one_video_dic)
            return list_page_Lst

    def search_page(self, keyword, search_pages_max=10, output_to_es_raw=
        False, output_to_es_register=False, es_index=None, doc_type=None):

        def process_one_line(data_line):
            title = data_line.find('div', {'class': 'v-meta-title'}).a['title']
            url = data_line.find('div', {'class': 'v-meta-title'}).a['href']
            releaser = data_line.find('span', {'class': 'username'}).text
            releaser = releaser.strip()
            releaserUrl = data_line.find('span', {'class': 'username'}).a[
                'href']
            releaserUrl = 'http:' + releaserUrl
            play_count = data_line.find('span', {'class': 'pub'}).text
            dura = data_line.find('span', {'class': 'v-time'}).text
            duration_str = dura
            dl = duration_str.split(':')
            dl_int = []
            for v in dl:
                v = int(v)
                dl_int.append(v)
            if len(dl_int) == 2:
                duration = dl_int[0] * 60 + dl_int[1]
            else:
                duration = dl_int[0] * 3660 + dl_int[1] * 60 + dl_int[2]
            videourl = 'https:' + url
            dict_page = self.video_page(videourl)
            release_time = dict_page['release_time']
            D0 = copy.deepcopy(self.video_data)
            D0['title'] = title
            D0['url'] = videourl
            D0['duration'] = duration
            D0['releaser'] = releaser
            D0['release_time'] = release_time
            D0['releaserUrl'] = releaserUrl
            D0['play_count'] = play_count
            return D0
        search_data_Lst = []
        jsurl = ('http://www.soku.com/search_video_ajax/q_' + keyword +
            '_orderby_1_limitdate_0?site=14&page={}'.format(str(i)) for i in
            range(2, 4))
        for urls in jsurl:
            print(urls)
            get_page = requests.get(urls)
            print('get jspage done')
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            youku = soup.find_all('div', {'class': 'v'})
            for data_line in youku:
                one_video_dic = process_one_line(data_line)
                search_data_Lst.append(one_video_dic)
        url = ('http://www.soku.com/search_video/q_' + keyword +
            '_orderby_1_limitdate_0?spm=a2h0k.8191407.0.0&site=14&page={}'.
            format(str(i)) for i in range(1, search_pages_max + 1))
        for urls in url:
            print(urls)
            get_page = requests.get(urls)
            print('get page done')
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            youku = soup.find_all('div', {'class': 'v'})
            for data_line in youku:
                one_video_dic = process_one_line(data_line)
                search_data_Lst.append(one_video_dic)
                if len(search_data_Lst) >= 100:
                    output_result(result_Lst=search_data_Lst, platform=self
                        .platform, output_to_es_raw=output_to_es_raw,
                        output_to_es_register=output_to_es_register,
                        es_index=es_index, doc_type=doc_type)
                    search_data_Lst.clear()
        if search_data_Lst != []:
            output_result(result_Lst=search_data_Lst, platform=self.
                platform, output_to_es_raw=output_to_es_raw,
                output_to_es_register=output_to_es_register, es_index=
                es_index, doc_type=doc_type)
        return search_data_Lst

    def get_releaser_detail(self, url):
        releaser_data_Lst = []
        get_page = requests.get(url)
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('a', {'class': 'username'})['title']
        releaser_intro = soup.find('span', {'class': 'desc'}).text
        total_play_count = soup.find('li', {'class': 'vnum'})['title']
        fans = soup.find('li', {'class': 'snum'})['title']
        midstep = soup.find_all('div', {'class': 'title'})
        for brother in midstep:
            if brother.h3.text == '视频':
                num = brother.find('span').text
                self.TotalVideo_num = num[1:-1]
        D_releaser_page = {'releaser': releaser, 'releaser_intro':
            releaser_intro, 'total_play_count': total_play_count, 'follow':
            fans, 'TotalVideo_num': self.TotalVideo_num}
        releaser_data_Lst.append(D_releaser_page)
        return D_releaser_page

    def get_releaser_info(self, releaserUrl):
        get_page = retry_get_url(releaserUrl)
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('a', {'class': 'username'})['title']
        releaser_id = soup.find('a', {'class': 'user-avatar'})['href']
        D0 = {'releaser': releaser, 'releaser_id': releaser_id}
        return D0

    @logged
    def releaser_page(self, releaserUrl, error_file=None, output_to_file=
        False, filepath=None, output_to_es_raw=False, output_es_index=None,
        output_doc_type=None, output_to_es_register=False, push_to_redis=
        False, releaser_page_num_max=300):
        result_Lst = []
        releaser_info = self.get_releaser_info(releaserUrl)
        releaser = releaser_info['releaser']
        releaser_id = releaser_info['releaser_id']
        count = 1
        while count <= releaser_page_num_max:
            releaser_url = ('https://i.youku.com' + releaser_id +
                '/videos?&order=1&page=' + str(count))
            count += 1
            get_page = retry_get_url(releaser_url)
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_lst = soup.find_all('div', {'class': 'v va'})
            for one_video in video_lst:
                try:
                    title = one_video.find('div', {'class': 'v-meta-title'}).a[
                        'title']
                    url = one_video.find('div', {'class': 'v-meta-title'}).a[
                        'href']
                    if url[0:4] != 'http':
                        url = 'https:' + url
                    try:
                        play_count_text = one_video.find('span', {'class':
                            'v-num'}).text
                        if '万' in play_count_text:
                            play_count_str = play_count_text.replace('万', '')
                            play_count = int(float(play_count_str) * 10000.0)
                        else:
                            play_count = int(play_count_text.replace(',', ''))
                    except:
                        play_count = 0
                    dura_str = one_video.find('span', {'class': 'v-time'}).text
                    dura_lst = dura_str.split(':')
                    if len(dura_lst) == 2:
                        duration = int(dura_lst[0]) * 60 + int(dura_lst[1])
                    elif len(dura_lst) == 3:
                        duration = int(dura_lst[0]) * 3600 + int(dura_lst[1]
                            ) * 60 + int(dura_lst[2])
                    else:
                        duration = 0
                    rt_str = one_video.find('span', {'class': 'v-publishtime'}
                        ).text
                    release_time = (trans_strtime_to_timestamp.
                        trans_strtime_to_timestamp(rt_str, missing_year=True))
                    fetch_time = int(datetime.datetime.timestamp(datetime.
                        datetime.now()) * 1000.0)
                    self.video_data['title'] = title
                    self.video_data['duration'] = duration
                    self.video_data['url'] = url
                    self.video_data['play_count'] = play_count
                    self.video_data['releaser'] = releaser
                    self.video_data['releaserUrl'] = releaserUrl
                    self.video_data['release_time'] = release_time
                    self.video_data['fetch_time'] = fetch_time
                    get_data = copy.deepcopy(self.video_data)
                    result_Lst.append(get_data)
                    print('get_one_line %s' % url)
                except:
                    print('failed to get data')
                    pass
                if len(result_Lst) % 100 == 0:
                    if output_doc_type is None and output_es_index is None:
                        output_result(result_Lst, self.platform,
                            output_to_file=output_to_file, filepath=
                            filepath, output_to_es_raw=output_to_es_raw,
                            output_to_es_register=output_to_es_register,
                            push_to_redis=push_to_redis)
                        result_Lst.clear()
                    elif output_doc_type is not None and output_es_index is not None:
                        output_result(result_Lst, self.platform,
                            output_to_file=output_to_file, filepath=
                            filepath, output_to_es_raw=output_to_es_raw,
                            output_to_es_register=output_to_es_register,
                            push_to_redis=push_to_redis, es_index=
                            output_es_index, doc_type=output_doc_type)
                        result_Lst.clear()
        if result_Lst != []:
            if output_doc_type is None and output_es_index is None:
                output_result(result_Lst, self.platform, output_to_file=
                    output_to_file, filepath=filepath, output_to_es_raw=
                    output_to_es_raw, output_to_es_register=
                    output_to_es_register, push_to_redis=push_to_redis)
            elif output_doc_type is not None and output_es_index is not None:
                output_result(result_Lst, self.platform, output_to_file=
                    output_to_file, filepath=filepath, output_to_es_raw=
                    output_to_es_raw, output_to_es_register=
                    output_to_es_register, push_to_redis=push_to_redis,
                    es_index=output_es_index, doc_type=output_doc_type)

    def doc_list(self, resultfile, donefile, doneurl, errorfile, area):
        list_data_lst = []
        result = open(resultfile, 'a')
        done = open(donefile, 'a')
        donepage = open(doneurl, 'a')
        error = open(errorfile, 'a')
        count = 1
        video_count = 30
        while video_count == 30 and count <= 30:
            url = ('http://list.youku.com/category/show/c_84_a_' + area +
                '_s_1_d_1_p_' + str(count) + '.html')
            get_page = requests.get(url)
            get_page.encoding = 'utf-8'
            page = get_page.text
            print(url)
            soup = BeautifulSoup(page, 'html.parser')
            midstep = soup.find_all('div', {'class': 'p-thumb'})
            video_count = len(midstep)
            done.write(area)
            done.write(str(count))
            done.write('\n')
            done.flush()
            count += 1
            for line in midstep:
                url_step = line.a['href']
                if url_step[:4] != 'http':
                    url = 'http:' + url_step
                else:
                    url = url_step
                try:
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    soup = BeautifulSoup(page, 'html.parser')
                    print(url)
                    try:
                        donepage.write(url)
                        donepage.write('\n')
                        donepage.flush()
                        album_name = soup.find('h2').a.text
                        album_rating = soup.find('span', {'class': 'score'}
                            ).text
                        sumvideo = soup.find('p', {'class': 'tvintro'}).text
                        sumvideo_int = int(re.findall('\\d+', sumvideo)[0])
                        if sumvideo_int <= 100:
                            midstep = soup.find('div', {'id': 'listitem_page1'}
                                )
                            first_video = midstep.find('div', {'class':
                                'item item-cover current'})
                            url = 'http:' + first_video.a['href']
                            title = first_video['title']
                            dura = first_video.find('span', {'class': 'c-time'}
                                ).text
                            dura_lst = dura.split(':')
                            if len(dura_lst) == 1:
                                duration = int(dura_lst[0])
                            elif len(dura_lst) == 2:
                                duration = int(dura_lst[0]) * 60 + int(dura_lst
                                    [1])
                            elif len(dura_lst) == 3:
                                duration = int(dura_lst[0]) * 3600 + int(
                                    dura_lst[1]) * 60 + int(dura_lst[2])
                            playcount_str = first_video.find('div', {
                                'class': 'status'}).text
                            playcount_str = playcount_str.replace('次播放', '')
                            playcount_str = playcount_str.replace(',', '')
                            thousand_10 = '万'
                            if thousand_10 in playcount_str:
                                playcount_str = playcount_str.replace('万', '')
                                play_count = int(float(playcount_str) * 10000.0
                                    )
                            else:
                                play_count = int(playcount_str)
                            fetch_time = int(datetime.datetime.timestamp(
                                datetime.datetime.now()) * 1000.0)
                            D0 = {'url': url, 'tilte': title, 'duartion':
                                duration, 'play_count': play_count,
                                'album_name': album_name, 'album_rating':
                                album_rating, 'fetch_time': fetch_time}
                            D1 = {'url': url, 'video_infor': D0}
                            list_data_lst.append(D1)
                            json_D1 = json.dumps(D1)
                            result.write(json_D1)
                            result.write('\n')
                            result.flush()
                            print('get one line')
                            other_video = midstep.find_all('div', {'class':
                                'item item-cover'})
                            for line in other_video:
                                url = 'http:' + first_video.a['href']
                                title = first_video['title']
                                dura = first_video.find('span', {'class':
                                    'c-time'}).text
                                dura_lst = dura.split(':')
                                if len(dura_lst) == 1:
                                    duration = int(dura_lst[0])
                                elif len(dura_lst) == 2:
                                    duration = int(dura_lst[0]) * 60 + int(
                                        dura_lst[1])
                                elif len(dura_lst) == 3:
                                    duration = int(dura_lst[0]) * 3600 + int(
                                        dura_lst[1]) * 60 + int(dura_lst[2])
                                playcount_str = first_video.find('div', {
                                    'class': 'status'}).text
                                playcount_str = playcount_str.replace('次播放', ''
                                    )
                                playcount_str = playcount_str.replace(',', '')
                                thousand_10 = '万'
                                if thousand_10 in playcount_str:
                                    playcount_str = playcount_str.replace('万',
                                        '')
                                    play_count = int(float(playcount_str) *
                                        10000.0)
                                else:
                                    play_count = int(playcount_str)
                                fetch_time = int(datetime.datetime.
                                    timestamp(datetime.datetime.now()) * 1000.0
                                    )
                                D0 = {'url': url, 'tilte': title,
                                    'duartion': duration, 'play_count':
                                    play_count, 'album_name': album_name,
                                    'album_rating': album_rating,
                                    'fetch_time': fetch_time}
                                D1 = {'url': url, 'video_infor': D0}
                                list_data_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                print('get one line')
                            result.flush()
                        if sumvideo_int > 100:
                            if sumvideo_int % 50 == 0:
                                page_num = int(sumvideo_int / 50)
                            else:
                                page_num = int(sumvideo_int / 50) + 1
                            videoid = re.findall('\\d+', re.findall(
                                'videoId:"[0-9]{1,10}"', page)[0])[0]
                            showid = re.findall('\\d+', re.findall(
                                'showid:"[0-9]{1,10}"', page)[0])[0]
                            url_lst = [(
                                'http://v.youku.com/page/playlist?&l=debug&pm=3&vid='
                                 + videoid + '&fid=0&showid=' + showid +
                                '&sid=0&page={}&callback=tuijsonp23'.format
                                (str(i))) for i in range(1, page_num + 1)]
                            for line in url_lst:
                                get_page = requests.get(line)
                                get_page.encoding = 'utf-8'
                                page = get_page.text
                                page_dic = eval(page[34:-2])
                                videolab = page_dic['html']
                                videolab = videolab.replace('\\', '')
                                soup = BeautifulSoup(videolab, 'html.parser')
                                first_video = soup.find('div', {'class':
                                    'item item-cover current'})
                                url = 'http:' + first_video.a['href']
                                title = first_video['title']
                                dura = first_video.find('span', {'class':
                                    'c-time'}).text
                                dura_lst = dura.split(':')
                                if len(dura_lst) == 1:
                                    duration = int(dura_lst[0])
                                elif len(dura_lst) == 2:
                                    duration = int(dura_lst[0]) * 60 + int(
                                        dura_lst[1])
                                elif len(dura_lst) == 3:
                                    duration = int(dura_lst[0]) * 3600 + int(
                                        dura_lst[1]) * 60 + int(dura_lst[2])
                                playcount_str = first_video.find('div', {
                                    'class': 'status'}).text
                                playcount_str = playcount_str.replace('次播放', ''
                                    )
                                playcount_str = playcount_str.replace(',', '')
                                thousand_10 = '万'
                                if thousand_10 in playcount_str:
                                    playcount_str = playcount_str.replace('万',
                                        '')
                                    play_count = int(float(playcount_str) *
                                        10000.0)
                                else:
                                    play_count = int(playcount_str)
                                fetch_time = int(datetime.datetime.
                                    timestamp(datetime.datetime.now()) * 1000.0
                                    )
                                D0 = {'url': url, 'tilte': title,
                                    'duartion': duration, 'play_count':
                                    play_count, 'album_name': album_name,
                                    'album_rating': album_rating,
                                    'fetch_time': fetch_time}
                                D1 = {'url': url, 'video_infor': D0}
                                list_data_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print('get one line')
                                other_video = midstep.find_all('div', {
                                    'class': 'item item-cover'})
                                for line in other_video:
                                    url = 'http:' + first_video.a['href']
                                    title = first_video['title']
                                    dura = first_video.find('span', {
                                        'class': 'c-time'}).text
                                    dura_lst = dura.split(':')
                                    if len(dura_lst) == 1:
                                        duration = int(dura_lst[0])
                                    elif len(dura_lst) == 2:
                                        duration = int(dura_lst[0]) * 60 + int(
                                            dura_lst[1])
                                    elif len(dura_lst) == 3:
                                        duration = int(dura_lst[0]) * 3600 + int(
                                            dura_lst[1]) * 60 + int(dura_lst[2])
                                    playcount_str = first_video.find('div',
                                        {'class': 'status'}).text
                                    playcount_str = playcount_str.replace('次播放'
                                        , '')
                                    playcount_str = playcount_str.replace(',',
                                        '')
                                    thousand_10 = '万'
                                    if thousand_10 in playcount_str:
                                        playcount_str = playcount_str.replace('万',
                                            '')
                                        play_count = int(float(playcount_str) *
                                            10000.0)
                                    else:
                                        play_count = int(playcount_str)
                                    fetch_time = int(datetime.datetime.
                                        timestamp(datetime.datetime.now()) *
                                        1000.0)
                                    D0 = {'url': url, 'tilte': title,
                                        'duartion': duration, 'play_count':
                                        play_count, 'album_name':
                                        album_name, 'album_rating':
                                        album_rating, 'fetch_time': fetch_time}
                                    D1 = {'url': url, 'video_infor': D0}
                                    list_data_lst.append(D1)
                                    json_D1 = json.dumps(D1)
                                    result.write(json_D1)
                                    result.write('\n')
                                    print('get one line')
                                result.flush()
                    except:
                        error.write(url)
                        error.write('\n')
                        error.flush()
                        print('there is an error')
                except:
                    error.write(url)
                    error.write('\n')
                    error.flush()
                    print('there is an error')
        done.close()
        result.close()
        return list_data_lst

    def add_infor(self, taskname, resultname):
        task = open(taskname, 'a')
        result = open(resultname, 'a')
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            video_infor = line_dic['video_infor']
            task_lst.append(video_infor)
        for line in task_lst:
            url = line['url']
            try:
                video_dic = self.video_page(url)
                line['releaser'] = video_dic['releaser']
                line['release_time'] = video_dic['release_time']
                line['releaserUrl'] = video_dic['releaserUrl']
                line['tags'] = video_dic['tags']
                json_line = json.dumps(line)
                result_lst.append(line)
                result.write(json_line)
                result.write('\n')
                result.flush()
            except:
                print('no video infor')
        result.close()
        return result_lst

    def get_doc(self, cluename, taskname, resultname, donename):
        task = open(taskname)
        result = open(resultname, 'a')
        clue = open(cluename, 'a')
        done = open(donename, 'a')
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls = line['url']
            urls = urls.replace('\n', '')
            album_name = line['album_name']
            album_rating = line['album_rating']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = ' '.join(re.findall('showid: &#\\d+;\\d+', page)
                    ).split(';')[1]
                videoId = ' '.join(re.findall('videoId: &#\\d+;\\d+', page)
                    ).split(';')[1]
                videoCategoryId = ' '.join(re.findall(
                    'videoCategoryId: &#\\d+;\\d+', page)).split(';')[1]
                componentid = ' '.join(re.findall('"componentId":\\d+', page)
                    ).split(':')[1]
                D0 = {'showid': showid, 'videoid': videoId,
                    'videocategoryid': videoCategoryId, 'componentid':
                    componentid}
                json_D0 = json.dumps(D0)
                clue.write(json_D0)
                clue.write('\n')
                clue.flush()
                count = 1
                thousand_10 = '万'
                url = (
                    'http://v.youku.com/page/playlist?&l=debug&pm=3&vid=' +
                    videoId + '&fid=0&showid=' + showid +
                    '&sid=0&componentid=' + componentid +
                    '&videoCategoryId=' + videoCategoryId +
                    '&isSimple=false&page=' + str(count))
                get_page = requests.get(url)
                get_page.encoding = 'utf-8'
                page = get_page.text
                page_dic = eval(page)
                video_infor = page_dic['html']
                soup = BeautifulSoup(video_infor, 'html.parser')
                video_dic = soup.find_all('div', {'class': 'item item-cover'})
                sumofvideo = len(video_dic)
                for line in video_dic:
                    title = line['title']
                    url = line.a['href']
                    if 'http:' not in url:
                        url = 'http:' + url
                    dura = line.find('span', {'class': 'c-time'}).text
                    dura_lst = dura.split(':')
                    if len(dura_lst) == 1:
                        duration = int(dura_lst[0])
                    elif len(dura_lst) == 2:
                        duration = int(dura_lst[0]) * 60 + int(dura_lst[1])
                    elif len(dura_lst) == 3:
                        duration = int(dura_lst[0]) * 3600 + int(dura_lst[1]
                            ) * 60 + int(dura_lst[2])
                    play_count_str = line.find('div', {'class': 'status'}).text
                    play_count_str = play_count_str.replace('次播放', '')
                    play_count_str = play_count_str.replace(',', '')
                    if thousand_10 in play_count_str:
                        play_count_str = play_count_str.replace('万', '')
                        play_count = int(float(play_count_str) * 10000.0)
                    else:
                        play_count = int(play_count_str)
                    fetch_time = int(datetime.datetime.timestamp(datetime.
                        datetime.now()) * 1000.0)
                    D1 = {'album_name': album_name, 'album_rating':
                        album_rating, 'title': title, 'duration': duration,
                        'play_count': play_count, 'url': url, 'fetch_time':
                        fetch_time}
                    if D1 not in result_lst:
                        result_lst.append(D1)
                        json_D1 = json.dumps(D1)
                        result.write(json_D1)
                        result.write('\n')
                        result.flush()
                        print(url)
                while sumofvideo == 100:
                    count += 1
                    url = (
                        'http://v.youku.com/page/playlist?&l=debug&pm=3&vid=' +
                        videoId + '&fid=0&showid=' + showid +
                        '&sid=0&componentid=' + componentid +
                        '&videoCategoryId=' + videoCategoryId +
                        '&isSimple=false&page=' + str(count))
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    page_dic = eval(page)
                    video_infor = page_dic['html']
                    if video_infor == '\n':
                        sumofvideo = 0
                    else:
                        soup = BeautifulSoup(video_infor, 'html.parser')
                        video_dic = soup.find_all('div', {'class':
                            'item item-cover'})
                        sumofvideo = len(video_dic)
                        for line in video_dic:
                            title = line['title']
                            url = line.a['href']
                            if 'http:' not in url:
                                url = 'http:' + url
                            dura = line.find('span', {'class': 'c-time'}).text
                            dura_lst = dura.split(':')
                            if len(dura_lst) == 1:
                                duration = int(dura_lst[0])
                            elif len(dura_lst) == 2:
                                duration = int(dura_lst[0]) * 60 + int(dura_lst
                                    [1])
                            elif len(dura_lst) == 3:
                                duration = int(dura_lst[0]) * 3600 + int(
                                    dura_lst[1]) * 60 + int(dura_lst[2])
                            play_count_str = line.find('div', {'class':
                                'status'}).text
                            play_count_str = play_count_str.replace('次播放', '')
                            play_count_str = play_count_str.replace(',', '')
                            if thousand_10 in play_count_str:
                                play_count_str = play_count_str.replace('万', ''
                                    )
                                play_count = int(float(play_count_str) * 
                                    10000.0)
                            else:
                                play_count = int(play_count_str)
                            fetch_time = int(datetime.datetime.timestamp(
                                datetime.datetime.now()) * 1000.0)
                            D1 = {'album_name': album_name, 'album_rating':
                                album_rating, 'title': title, 'duration':
                                duration, 'play_count': play_count, 'url':
                                url, 'fetch_time': fetch_time}
                            if D1 not in result_lst:
                                result_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print(url)
                done.write(urls)
                done.write('\n')
                done.flush()
            except:
                pass
        done.close()
        task.close()
        result.close()
        clue.close()
        return result_lst

    def one_video(self, taskname):
        task = open(taskname)
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls = line['url']
            urls = urls.replace('\n', '')
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = ' '.join(re.findall('showid: &#\\d+;\\d+', page)
                    ).split(';')[1]
                print('more video')
            except:
                result_lst.append(urls)
                print('one video')
        task.close()
        return result_lst

    def get_releaser(self, taskname, resultname, errorname):
        task = open(taskname)
        result = open(resultname, 'a')
        error = open(errorname, 'a')
        task_lst = []
        result_lst = []
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls = line['url']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            page2 = page.replace('\\', '')
            try:
                releaser = re.findall('[一-龥]+', ''.join(re.findall(
                    'class="" title="[一-龥]+"', page2)))[0]
                print('get releaser')
            except:
                releaser = None
                error.write(urls)
                error.write('\n')
                error.flush()
                print('no releaser')
            line['releaser'] = releaser
            result_lst.append(line)
            json_line = json.dumps(line)
            result.write(json_line)
            result.write('\n')
            result.flush()
        result.close()
        return result_lst

    def get_release_time(self, url):
        get_page = requests.get(url)
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        try:
            time_one = soup.find('span', {'class': 'bold mr3'}).text
        except AttributeError:
            time_one = None
        try:
            time_two = time_one[4:]
        except TypeError:
            time_two = None
        try:
            midsteptime = time_two.strip()
        except AttributeError:
            midsteptime = None
        try:
            release_time = int(datetime.datetime.strptime(midsteptime,
                '%Y-%m-%d').timestamp() * 1000.0)
        except (TypeError, ValueError):
            release_time = None
        D_video_page = {'release_time': release_time, 'url': url}
        print('get one line')
        return D_video_page

    def get_video_play_count_releaser_info(self, url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        try:
            video_info = soup.find('div', {'class': 'item item-cover current'})
            play_count_str = video_info.find('div', {'class': 'status'}).text
            play_count_str = play_count_str.replace('次播放', '')
            if '万' in play_count_str:
                play_count_str = play_count_str.replace('万', '')
                play_count = int(float(play_count_str) * 10000.0)
            else:
                play_count = int(play_count_str.replace(',', ''))
        except:
            play_count = 0
        try:
            releaser = soup.find('span', {'id': 'module_basic_sub'}).a.text
            releaser = releaser.replace('\n', '')
            releaser = releaser.replace(' ', '')
            releaserUrl = soup.find('span', {'id': 'module_basic_sub'}).a[
                'href']
            if releaserUrl[0:4] == 'http':
                pass
            else:
                releaserUrl = 'http:' + releaserUrl
        except:
            releaser = None
            releaserUrl = None
        D0 = {'play_count': play_count, 'releaser': releaser, 'releaserUrl':
            releaserUrl}
        return D0

    def get_release_time_from_str(self, rt_str):
        minute = '分钟'
        hour = '小时'
        yesterday = '昨天'
        the_day_before_yesterday = '前天'
        the_day_before_many_days = '天前'
        if minute in rt_str or hour in rt_str:
            year = datetime.date.today().year
            month = datetime.date.today().month
            day = datetime.date.today().day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        elif yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days=1)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        elif the_day_before_yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days=2)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        elif the_day_before_many_days in rt_str:
            day_shift = int(re.findall('\\d+', rt_str)[0])
            raw_data = datetime.date.today() - datetime.timedelta(days=
                day_shift)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(
                day)
            release_time = int(datetime.datetime.strptime(date, '%Y-%m-%d')
                .timestamp() * 1000.0)
        else:
            release_time = 0
        return release_time

    def list_page_all(self, output_to_file=False, filepath=None,
        output_to_es_raw=False, output_to_es_register=False, push_to_redis=
        False, page_num_max=25):
        """
        Get all videos in youku.com '片库' in the home page bottom and the '全部'
        category with '最新发布' filter.
        At present (2018-06-28) the url is http://list.youku.com/category/video/c_0_d_1_s_2.html?spm=a2h1n.8251846.selectID.5!2~5~5!2~1~3!2~A
        This url changes to the following format
        'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str(count) + '.html'
        where count is page number.
        """
        if page_num_max > 25:
            page_num_max = 25
        result_Lst = []
        page_num = 1
        while page_num <= page_num_max:
            lst_url = (
                'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str
                (page_num) + '.html')
            print('Getting data on page %d: %s, %s' % (page_num, lst_url,
                datetime.datetime.now()))
            get_page = retry_get_url(lst_url)
            if get_page is None:
                print(
                    'Failed to get page url: %s, will continue to next page.' %
                    lst_url)
                continue
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_agg:
                title = line.find('li', {'class': 'title'}).text
                dura_str = line.find('li', {'class': 'status'}).text
                dura_lst = dura_str.split(':')
                if len(dura_lst) == 3:
                    duration = int(int(dura_lst[0]) * 3600 + int(dura_lst[1
                        ]) * 60 + int(dura_lst[2]))
                elif len(dura_lst) == 2:
                    duration = int(int(dura_lst[0]) * 60 + int(dura_lst[1]))
                else:
                    duration = 0
                url = line.a['href']
                if url[0:4] != 'http':
                    url = 'https:' + url
                video_info_dic = self.get_video_play_count_releaser_info(url)
                fetch_time = int(datetime.datetime.timestamp(datetime.
                    datetime.now()) * 1000.0)
                rt_str = line.find('li', {'class': ' '}).text
                release_time = self.get_release_time_from_str(rt_str)
                self.video_data['title'] = title
                self.video_data['duration'] = duration
                self.video_data['url'] = url
                self.video_data['play_count'] = video_info_dic['play_count']
                self.video_data['releaser'] = video_info_dic['releaser']
                self.video_data['releaserUrl'] = video_info_dic['releaserUrl']
                self.video_data['release_time'] = release_time
                self.video_data['fetch_time'] = fetch_time
                get_data = copy.deepcopy(self.video_data)
                result_Lst.append(get_data)
                if len(result_Lst) >= 100:
                    print('Got %d lines, %s' % (len(result_Lst), datetime.
                        datetime.now()))
                    output_result(result_Lst, self.platform, output_to_file
                        =output_to_file, filepath=filepath,
                        output_to_es_raw=output_to_es_raw,
                        output_to_es_register=output_to_es_register,
                        push_to_redis=push_to_redis)
                    result_Lst.clear()
            page_num += 1
        if result_Lst != []:
            print('Got %d lines, %s' % (len(result_Lst), datetime.datetime.
                now()))
            output_result(result_Lst, self.platform, output_to_file=
                output_to_file, filepath=filepath, output_to_es_raw=
                output_to_es_raw, output_to_es_register=
                output_to_es_register, push_to_redis=push_to_redis)
            result_Lst.clear()

    def get_title_subtitle(self, url):
        get_page = retry_get_url(url)
        if get_page is None:
            print('Failed to get url: %s' % url)
            return
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        try:
            subtitle = soup.find('span', {'id': 'subtitle'}).text
        except:
            subtitle = None
        try:
            title = soup.h1.find('a', {'target': '_blank'}).text
            title_str = title + subtitle
        except:
            title_str = subtitle
        return title_str

    def select_tv_series_video(self, sp_video_album_name, result_file=None,
        output_to_file=False, filepath=None, output_to_es_raw=False,
        output_to_es_register=False, push_to_redis=False):
        select_result = []
        meta = Meta()
        result_lst = meta.dic_file_to_lst(result_file)
        for line in result_lst:
            url = line['url']
            title_str = self.get_title_subtitle(url)
            if sp_video_album_name in title_str:
                line.update({'title': title_str})
                select_result.append(line)
                print('it_belongs_to_what_we_want')
                if len(select_result) >= 100:
                    print('Got %d lines, %s' % (len(select_result),
                        datetime.datetime.now()))
                    output_result(select_result, self.platform,
                        output_to_file=output_to_file, filepath=filepath,
                        output_to_es_raw=output_to_es_raw,
                        output_to_es_register=output_to_es_register,
                        push_to_redis=push_to_redis)
                    select_result.clear()
            else:
                print('not_belongs_to')
        if select_result != []:
            print('Got %d lines, %s' % (len(select_result), datetime.
                datetime.now()))
            output_result(select_result, self.platform, output_to_file=
                output_to_file, filepath=filepath, output_to_es_raw=
                output_to_es_raw, output_to_es_register=
                output_to_es_register, push_to_redis=push_to_redis)
            select_result.clear()

    def start_list_page(self, task_list, output_to_file=False, filepath=
        None, output_to_es_raw=False, push_to_redis=False, es_index=
        'crawler-data-raw', doc_type='doc', output_to_es_register=False):
        self.list_page_task(task_list)
        self.download_list_page_single_process()

    def list_page_task(self, task_list, page_num_max=25):
        TASK_LIST = []
        for line in task_list:
            for num in range(1, page_num_max + 1):
                url = line.replace('fangyucheng', str(num))
                TASK_LIST.append(url)
        random.shuffle(TASK_LIST)
        connect_with_redis.push_list_url_to_redis(platform=self.platform,
            result_lst=TASK_LIST)

    async def download_page(self, session, url):
        get_page = await session.get(url)
        page = await get_page.text('utf-8', errors='ignore')
        return page

    async def download_list_page(self, loop):
        task_list = connect_with_redis.retrieve_list_url_from_redis(platform
            =self.platform)
        async with aiohttp.ClientSession() as session:
            task = [loop.create_task(self.download_page(session, url)) for
                url in task_list]
            done, pending = await asyncio.wait(task)
            RESULT_LIST = [d.result() for d in done]
            connect_with_redis.push_list_page_html_to_redis(platform=self.
                platform, result_lst=RESULT_LIST)

    def download_list_page_single_process(self):
        key = 'youku_list_url'
        while connect_with_redis.length_of_lst(key) > 0:
            loop = asyncio.get_event_loop()
            loop.run_until_complete(self.download_list_page(loop))

    def parse_list_page_single_process(self):
        pid = os.getpid()
        URL_LIST = []
        key = 'youku_list_page_html'
        while connect_with_redis.length_of_lst(key) > 0:
            PAGE = connect_with_redis.retrieve_list_page_html_from_redis(
                platform=self.platform)
            soup = BeautifulSoup(PAGE, 'html.parser')
            video_url_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_url_agg:
                url = line.a['href']
                video_id_str = ' '.join(re.findall('id_.*html', url))
                video_id = video_id_str.replace('id_', '')
                tudou_url = 'https://video.tudou.com/v/%s' % video_id
                URL_LIST.append(tudou_url)
                connect_with_redis.push_video_url_to_redis_set(platform=
                    'new_tudou', url_lst=URL_LIST)
            print(
                'platform: %s, action: parse list page, process_id: %s, get %s urls'
                 % (self.platform, pid, len(URL_LIST)))

    def parse_list_page_multi_process(self, processes_num=20):
        pool = Pool(processes=processes_num)
        for num in range(processes_num):
            pool.apply_async(self.parse_list_page_single_process)
        pool.close()
        pool.join()


<mask token>
if __name__ == '__main__':
    test = Crawler_youku()
    vid = 'XMjc5MTQyNjg0OA'
    test_selenium = test.get_video_play_count_by_vid(vid)
    sr_yk = test.search_page(keyword='任正非 BBC', search_pages_max=2)

# -*- coding: utf-8 -*-
"""
Created on Sat Feb 24 14:19:00 2018

@author: fangyucheng
"""

import os
import time
import aiohttp
import asyncio
import requests
from bs4 import BeautifulSoup
import datetime
import re
import json
import copy
import random
# from selenium import webdriver
from multiprocessing import Pool
from multiprocessing import Process
from crawler_sys.framework.video_fields_std import Std_fields_video
from crawler_sys.utils.output_results import retry_get_url
from crawler_sys.utils.output_results import output_result
from crawler_sys.utils import trans_strtime_to_timestamp
from crawler.crawler_sys.utils import connect_with_redis
from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
from crawler.crawler_sys.utils.util_logging import logged


class Crawler_youku():

    def __init__(self, timeout=None, platform='youku'):
        if timeout==None:
            self.timeout = 10
        else:
            self.timeout = timeout
        self.platform = platform
        std_fields = Std_fields_video()
        self.video_data = std_fields.video_data
        self.video_data['platform'] = self.platform
        # remove fields that crawled data don't have
        pop_key_Lst = ['channel', 'describe', 'repost_count', 'isOriginal',
                       'video_id']
        for popk in pop_key_Lst:
            self.video_data.pop(popk)

        # with channel == '全部', url field is hard coded in list_page_all method,
        # input url will be ignored
        self.legal_channels = ['全部']
        self.list_page_url_dict = {
            '全部': '_',
            }

    def video_page(self, url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('span',{'id': "module_basic_sub"}).a.text
        releaser = releaser.replace('\n', '')
        releaser = releaser.replace(' ', '')
        releaserUrl = soup.find('span',{'id': "module_basic_sub"}).a['href']
        if releaserUrl[0:4] == 'http':
            pass
        else:
            releaserUrl = 'http:' + releaserUrl
        try:
            rt_str = soup.find('span', {'class':'video-status'}).text
            rt_lst = re.findall('\d+', rt_str)
            rt_to_timestamp = str(rt_lst[0]) + '-' + str(rt_lst[1]) + '-' + str(rt_lst[2])
            release_time = int(datetime.datetime.strptime(rt_to_timestamp, '%Y-%m-%d').timestamp()*1e3)
        except:
            release_time = 0
        D0 = {'releaser':releaser,'releaserUrl':releaserUrl,'release_time':release_time}
        return D0

    def get_video_infor_by_network(self,url_Lst):
        video_id_Lst = []
        for url in url_Lst:
            video_id = ' '.join(re.findall('id.*html',url)).split('_')[1].split('.')[0]
            video_id_Lst.append(video_id)


    def get_video_title_play_count_duration(self, url):
        video_id = ' '.join(re.findall('id.*html', url))
        mobile_url = 'https://m.youku.com/video/' + video_id
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        title = browser.find_element_by_class_name('online').text
        dura = browser.find_element_by_class_name('v-label').text.split('\n')[1]
        dura_lst = ' '.join(dura).split[':']
        if len(dura_lst) == 2:
            duration = dura_lst[0]*60 + dura_lst[1]
        elif len(dura_lst) == 3:
            duration = dura_lst[0]*3600 + dura_lst[1]*60 + dura_lst[2]
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',','')
        pc_midstep4 = re.findall('[0-9]+',pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0]*1e4+pc_midstep4[1]*1e3
        else:
            play_count = ' '.join(pc_midstep4)
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
        D0 = {'video_id': video_id,
              'title': title,
              'duration': duration,
              'play_count': play_count,
              'url': url,
              'fetch_time': fetch_time}
        browser.close()
        return D0

    def get_video_play_count_by_vid(self, vid):
        mobile_url = 'https://m.youku.com/video/id_' + vid + '==.html'
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        time.sleep(10)
        title = browser.find_element_by_class_name('online').text
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',','')
        pc_midstep4 = re.findall('[0-9]+', pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0]*1e4+pc_midstep4[1]*1e3
        else:
            play_count = ' '.join(pc_midstep4)
        browser.close()
        D0 = {'title': title,
              'play_count': play_count}
        return D0

    def get_video_title_releaser_release_time(self, url):
        """An example of selenium
        """
        video_id = ' '.join(re.findall('id.*html', url))
        browser = webdriver.Chrome()
        browser.get(url)
        title = browser.find_element_by_id('subtitle').text
        releaser = browser.find_element_by_id('module_basic_sub').text
        releaser = releaser.replace('+订阅','')
        releaser = releaser.replace(' ','')
        try:
            rt_midstep = browser.find_element_by_class_name('video-status').text
            rt_midstep = rt_midstep.replace('上传于','')
            rt_midstep = rt_midstep.replace(' ','')
            release_time = int(datetime.datetime.strptime(rt_midstep,'%Y-%m-%d').timestamp()*1e3)
        except:
            release_time = 0
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
        D0 = {'video_id': video_id,
              'title': title,
              'release_time': release_time,
              'url': url,
              'fetch_time': fetch_time}
        return D0

    def list_page_sync(self, list_page_url, channel=None,
                  output_to_file=False, filepath=None,
                  output_to_es_raw=False,
                  output_to_es_register=False,
                  push_to_redis=False,
                  page_num_max=30):
        if channel == '全部':
            self.list_page_all(output_to_file=output_to_file, filepath=filepath,
                               output_to_es_raw=output_to_es_raw,
                               output_to_es_register=output_to_es_register,
                               push_to_redis=push_to_redis,
                               page_num_max=page_num_max)
        else:
            # need to be tested
            get_page = requests.get(list_page_url)
            page = get_page.text
            soup = BeautifulSoup(page,'html.parser')
            video = soup.find_all('div',{'class':'v-link'})
            dict_url = []
            for ttt in video:
                midstepurl=ttt.find('a')['href']
                if ('https:' in midstepurl):
                    midstepurl=midstepurl[6:]
                url = 'http:'+midstepurl
                dict_url.append(url)
            for xxxx in dict_url:
                if ("id" not in xxxx):
                    num=dict_url.index(xxxx)
                    dict_url.pop(num)
            for xxxx in dict_url:
                if ("id" not in xxxx):
                    num=dict_url.index(xxxx)
                    dict_url.pop(num)
            list_page_Lst = []
            for videourl in dict_url:
                one_video_dic = self.video_page(videourl)
                print ('one page done')
                list_page_Lst.append(one_video_dic)
            return list_page_Lst


    def search_page(self, keyword, search_pages_max=10,
                    output_to_es_raw=False,
                    output_to_es_register=False,
                    es_index=None,
                    doc_type=None):
        def process_one_line(data_line):
            title = data_line.find("div", {"class" : "v-meta-title"}).a['title']
            url = data_line.find("div", {"class" : "v-meta-title"}).a['href']
            releaser = data_line.find("span", {"class" : "username"}).text
            releaser = releaser.strip()
            releaserUrl = data_line.find("span", {"class" : "username"}).a['href']
            releaserUrl = 'http:' + releaserUrl
            play_count = data_line.find("span", {"class" : "pub"}).text
            dura = data_line.find("span", {"class" : "v-time"}).text
            duration_str = dura
            dl = duration_str.split(':')
            dl_int = []
            for v in dl:
                v = int(v)
                dl_int.append(v)
            if len(dl_int) == 2:
                duration = dl_int[0]*60+dl_int[1]
            else:
                duration = dl_int[0]*3660 + dl_int[1]*60+dl_int[2]
            videourl = 'https:' + url
            dict_page = self.video_page(videourl)
            release_time = dict_page['release_time']
            D0 = copy.deepcopy(self.video_data)
            D0['title'] = title
            D0['url'] = videourl
            D0['duration'] = duration
            D0['releaser'] = releaser
            D0['release_time'] = release_time
            D0['releaserUrl'] = releaserUrl
            D0['play_count'] = play_count

            return D0

        search_data_Lst = []
        jsurl = ('http://www.soku.com/search_video_ajax/q_'
                 + keyword
                 + '_orderby_1_limitdate_0?site=14&page={}'.format(
                     str(i)) for i in range (2, 4))
        for urls in jsurl:
            print(urls)
            get_page = requests.get(urls)
            print('get jspage done')
            page = get_page.text
            soup = BeautifulSoup(page,'html.parser')
            youku = soup.find_all("div", { "class" : "v" })
            for data_line in youku:
                one_video_dic = process_one_line(data_line)
                search_data_Lst.append(one_video_dic)
        url=('http://www.soku.com/search_video/q_'
             + keyword
             + '_orderby_1_limitdate_0?spm=a2h0k.8191407.0.0&site=14&page={}'.format(
                 str(i)) for i in range(1, search_pages_max+1))
        for urls in url:
            print(urls)
            get_page = requests.get(urls)
            print('get page done')
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            youku = soup.find_all("div", {"class" : "v"})
            for data_line in youku:
                one_video_dic = process_one_line(data_line)
                search_data_Lst.append(one_video_dic)
                if len(search_data_Lst) >= 100:
                    output_result(result_Lst=search_data_Lst,
                                  platform=self.platform,
                                  output_to_es_raw=output_to_es_raw,
                                  output_to_es_register=output_to_es_register,
                                  es_index=es_index,
                                  doc_type=doc_type)
                    search_data_Lst.clear()

        if search_data_Lst != []:
            output_result(result_Lst=search_data_Lst,
                          platform=self.platform,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          es_index=es_index,
                          doc_type=doc_type)

        return search_data_Lst


    def get_releaser_detail(self, url):
        releaser_data_Lst = []
        get_page = requests.get(url)
        page = get_page.text
        soup = BeautifulSoup(page,'html.parser')
        releaser=soup.find('a',{'class':'username'})['title']
        releaser_intro=soup.find('span',{'class':'desc'}).text
        total_play_count=soup.find('li',{'class':'vnum'})['title']
        fans=soup.find('li',{'class':'snum'})['title']
        midstep=soup.find_all('div',{'class':'title'})
        for brother in midstep:
            if brother.h3.text=='视频':
                num=brother.find('span').text
                self.TotalVideo_num=num[1:-1]
        D_releaser_page={"releaser":releaser,"releaser_intro":releaser_intro,"total_play_count":total_play_count,"follow":fans,"TotalVideo_num":self.TotalVideo_num}
        releaser_data_Lst.append(D_releaser_page)
        return D_releaser_page


    def get_releaser_info(self, releaserUrl):
        get_page = retry_get_url(releaserUrl)
        page = get_page.text
        soup = BeautifulSoup(page,'html.parser')
        releaser = soup.find('a',{'class':'username'})['title']
        releaser_id = soup.find('a',{'class':'user-avatar'})['href']
        #total_video_num = soup.find('div',{'class':'title'}).text
        D0 = {'releaser':releaser, 'releaser_id':releaser_id}
        return D0

    @logged
    def releaser_page(self, releaserUrl, error_file=None,
                      output_to_file=False, filepath=None,
                      output_to_es_raw=False,
                      output_es_index=None,
                      output_doc_type=None,
                      output_to_es_register=False,
                      push_to_redis=False,
                      releaser_page_num_max=300):
        result_Lst = []
        releaser_info = self.get_releaser_info(releaserUrl)
        releaser = releaser_info['releaser']
        releaser_id = releaser_info['releaser_id']
        count = 1
        while count <= releaser_page_num_max:
            releaser_url = 'https://i.youku.com' + releaser_id + '/videos?&order=1&page=' + str(count)
            count += 1
            get_page = retry_get_url(releaser_url)
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_lst = soup.find_all("div", { "class" : "v va" })
            for one_video in video_lst:
                try:
                    title = one_video.find("div",{"class":"v-meta-title"}).a['title']
                    url = one_video.find('div',{'class':'v-meta-title'}).a['href']
                    if url[0:4] != 'http':
                        url = 'https:' + url
                    try:
                        play_count_text = one_video.find("span",{"class":"v-num"}).text
                        if '万' in play_count_text:
                            play_count_str = play_count_text.replace('万', '')
                            play_count = int(float(play_count_str)*1e4)
                        else:
                            play_count = int(play_count_text.replace(',', ''))
                    except:
                        play_count = 0
                    dura_str = one_video.find("span",{"class":"v-time"}).text
                    dura_lst = dura_str.split(':')
                    if len(dura_lst) == 2:
                        duration = int(dura_lst[0])*60 + int(dura_lst[1])
                    elif len(dura_lst) == 3:
                        duration = int(dura_lst[0])*3600 + int(dura_lst[1])*60 + int(dura_lst[2])
                    else:
                        duration = 0
                    rt_str = one_video.find('span',{'class':'v-publishtime'}).text
                    release_time = trans_strtime_to_timestamp.trans_strtime_to_timestamp(rt_str, missing_year=True)
                    fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)

                    self.video_data['title'] = title
                    self.video_data['duration'] = duration
                    self.video_data['url'] = url
                    self.video_data['play_count'] = play_count
                    self.video_data['releaser'] = releaser
                    self.video_data['releaserUrl'] = releaserUrl
                    self.video_data['release_time'] = release_time
                    self.video_data['fetch_time'] = fetch_time

                    get_data = copy.deepcopy(self.video_data)
                    result_Lst.append(get_data)
                    print('get_one_line %s' % url)
                except:
                    print('failed to get data')
                    pass
                if len(result_Lst)%100 == 0:
                    if output_doc_type is None and output_es_index is None:
                        output_result(result_Lst,
                                      self.platform,
                                      output_to_file=output_to_file,
                                      filepath=filepath,
                                      output_to_es_raw=output_to_es_raw,
                                      output_to_es_register=output_to_es_register,
                                      push_to_redis=push_to_redis)
                        result_Lst.clear()

                    elif output_doc_type is not None and output_es_index is not None:
                        output_result(result_Lst,
                                      self.platform,
                                      output_to_file=output_to_file,
                                      filepath=filepath,
                                      output_to_es_raw=output_to_es_raw,
                                      output_to_es_register=output_to_es_register,
                                      push_to_redis=push_to_redis,
                                      es_index=output_es_index,
                                      doc_type=output_doc_type)
                        result_Lst.clear()

        if result_Lst!=[]:
            if output_doc_type is None and output_es_index is None:
                output_result(result_Lst,
                              self.platform,
                              output_to_file=output_to_file,
                              filepath=filepath,
                              output_to_es_raw=output_to_es_raw,
                              output_to_es_register=output_to_es_register,
                              push_to_redis=push_to_redis)
            elif output_doc_type is not None and output_es_index is not None:
                output_result(result_Lst,
                              self.platform,
                              output_to_file=output_to_file,
                              filepath=filepath,
                              output_to_es_raw=output_to_es_raw,
                              output_to_es_register=output_to_es_register,
                              push_to_redis=push_to_redis,
                              es_index=output_es_index,
                              doc_type=output_doc_type)


    def doc_list(self, resultfile,donefile,doneurl,errorfile,area):
        list_data_lst=[]
        result=open(resultfile,'a')
        done=open(donefile,'a')
        donepage=open(doneurl,'a')
        error=open(errorfile,'a')
        count=1
        video_count=30
        while video_count==30 and count<=30:
            url='http://list.youku.com/category/show/c_84_a_'+area+'_s_1_d_1_p_'+str(count)+'.html'
            get_page=requests.get(url)
            get_page.encoding='utf-8'
            page = get_page.text
            print(url)
            soup = BeautifulSoup(page,'html.parser')
            midstep = soup.find_all('div',{'class':'p-thumb'})
            video_count=len(midstep)
            done.write(area)
            done.write(str(count))
            done.write('\n')
            done.flush()
            count+=1
            for line in midstep:
                url_step=line.a['href']
                if url_step[:4]!='http':
                    url='http:'+url_step
                else:
                    url=url_step
                try:
                    get_page=requests.get(url)
                    get_page.encoding='utf-8'
                    page = get_page.text
                    soup = BeautifulSoup(page,'html.parser')
                    print(url)
                    try:
                        donepage.write(url)
                        donepage.write('\n')
                        donepage.flush()
                        album_name=soup.find('h2').a.text
                        album_rating=soup.find('span',{'class':'score'}).text
                        sumvideo=soup.find('p',{'class':'tvintro'}).text
                        sumvideo_int=int(re.findall('\d+',sumvideo)[0])
                        if sumvideo_int<=100:
                            midstep=soup.find('div',{'id':'listitem_page1'})
                            first_video=midstep.find('div',{'class':'item item-cover current'})
                            url='http:'+first_video.a['href']
                            title=first_video['title']
                            dura=first_video.find('span',{'class':'c-time'}).text
                            dura_lst=dura.split(':')
                            if len(dura_lst)==1:
                                duration=int(dura_lst[0])
                            elif len(dura_lst)==2:
                                duration=int(dura_lst[0])*60+int(dura_lst[1])
                            elif len(dura_lst)==3:
                                duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                            playcount_str=first_video.find('div',{'class':'status'}).text
                            playcount_str=playcount_str.replace('次播放','')
                            playcount_str=playcount_str.replace(',','')
                            thousand_10='万'
                            if thousand_10 in playcount_str:
                                playcount_str=playcount_str.replace('万','')
                                play_count=int((float(playcount_str))*1e4)
                            else:
                                play_count=int(playcount_str)
                            fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                            D0={'url':url,'tilte':title,'duartion':duration,
                                'play_count':play_count,'album_name':album_name,
                                'album_rating':album_rating,'fetch_time':fetch_time}
                            D1={'url':url,'video_infor':D0}
                            list_data_lst.append(D1)
                            json_D1=json.dumps(D1)
                            result.write(json_D1)
                            result.write('\n')
                            result.flush()
                            print('get one line')
                            other_video=midstep.find_all('div',{'class':'item item-cover'})
                            for line in other_video:
                                url='http:'+first_video.a['href']
                                title=first_video['title']
                                dura=first_video.find('span',{'class':'c-time'}).text
                                dura_lst=dura.split(':')
                                if len(dura_lst)==1:
                                    duration=int(dura_lst[0])
                                elif len(dura_lst)==2:
                                    duration=int(dura_lst[0])*60+int(dura_lst[1])
                                elif len(dura_lst)==3:
                                    duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                                playcount_str=first_video.find('div',{'class':'status'}).text
                                playcount_str=playcount_str.replace('次播放','')
                                playcount_str=playcount_str.replace(',','')
                                thousand_10='万'
                                if thousand_10 in playcount_str:
                                    playcount_str=playcount_str.replace('万','')
                                    play_count=int((float(playcount_str))*1e4)
                                else:
                                    play_count=int(playcount_str)
                                fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                                D0={'url':url,'tilte':title,'duartion':duration,
                                'play_count':play_count,'album_name':album_name,
                                'album_rating':album_rating,'fetch_time':fetch_time}
                                D1={'url':url,'video_infor':D0}
                                list_data_lst.append(D1)
                                json_D1=json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                print('get one line')
                            result.flush()
                        if sumvideo_int>100:
                            if sumvideo_int%50==0:
                                page_num=int((sumvideo_int/50))
                            else:
                                page_num=int((sumvideo_int)/50)+1
                            videoid=re.findall('\d+',(re.findall('videoId:"[0-9]{1,10}"',page)[0]))[0]
                            showid=re.findall('\d+',(re.findall('showid:"[0-9]{1,10}"',page)[0]))[0]
                            url_lst=['http://v.youku.com/page/playlist?&l=debug&pm=3&vid='+videoid+
                                     '&fid=0&showid='+showid+'&sid=0&page={}&callback=tuijsonp23'
                                     .format(str(i)) for i in range(1,page_num+1)]
                            for line in url_lst:
                                get_page = requests.get(line)
                                get_page.encoding = 'utf-8'
                                page = get_page.text
                                page_dic = eval(page[34:-2])
                                videolab = page_dic['html']
                                videolab = videolab.replace('\\','')
                                soup = BeautifulSoup(videolab,'html.parser')
                                first_video=soup.find('div',{'class':'item item-cover current'})
                                url='http:'+first_video.a['href']
                                title=first_video['title']
                                dura=first_video.find('span',{'class':'c-time'}).text
                                dura_lst=dura.split(':')
                                if len(dura_lst)==1:
                                    duration=int(dura_lst[0])
                                elif len(dura_lst)==2:
                                    duration=int(dura_lst[0])*60+int(dura_lst[1])
                                elif len(dura_lst)==3:
                                    duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                                playcount_str=first_video.find('div',{'class':'status'}).text
                                playcount_str=playcount_str.replace('次播放','')
                                playcount_str=playcount_str.replace(',','')
                                thousand_10='万'
                                if thousand_10 in playcount_str:
                                    playcount_str=playcount_str.replace('万','')
                                    play_count=int((float(playcount_str))*1e4)
                                else:
                                    play_count=int(playcount_str)
                                fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                                D0={'url':url,'tilte':title,'duartion':duration,
                                    'play_count':play_count,'album_name':album_name,
                                    'album_rating':album_rating,'fetch_time':fetch_time}
                                D1={'url':url,'video_infor':D0}
                                list_data_lst.append(D1)
                                json_D1=json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print('get one line')
                                other_video=midstep.find_all('div',{'class':'item item-cover'})
                                for line in other_video:
                                    url='http:'+first_video.a['href']
                                    title=first_video['title']
                                    dura=first_video.find('span',{'class':'c-time'}).text
                                    dura_lst=dura.split(':')
                                    if len(dura_lst)==1:
                                        duration=int(dura_lst[0])
                                    elif len(dura_lst)==2:
                                        duration=int(dura_lst[0])*60+int(dura_lst[1])
                                    elif len(dura_lst)==3:
                                        duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                                    playcount_str=first_video.find('div',{'class':'status'}).text
                                    playcount_str=playcount_str.replace('次播放','')
                                    playcount_str=playcount_str.replace(',','')
                                    thousand_10='万'
                                    if thousand_10 in playcount_str:
                                        playcount_str=playcount_str.replace('万','')
                                        play_count=int((float(playcount_str))*1e4)
                                    else:
                                        play_count=int(playcount_str)
                                    fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                                    D0={'url':url,'tilte':title,'duartion':duration,'play_count':play_count,
                                        'album_name':album_name,'album_rating':album_rating,'fetch_time':fetch_time}
                                    D1={'url':url,'video_infor':D0}
                                    list_data_lst.append(D1)
                                    json_D1=json.dumps(D1)
                                    result.write(json_D1)
                                    result.write('\n')
                                    print('get one line')
                                result.flush()
                    except:
                        error.write(url)
                        error.write('\n')
                        error.flush()
                        print('there is an error')
                except:
                    error.write(url)
                    error.write('\n')
                    error.flush()
                    print('there is an error')
        done.close()
        result.close()
        return list_data_lst

    def add_infor(self,taskname,resultname):
        task=open(taskname,'a')
        result=open(resultname,'a')
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic=eval(line)
            video_infor=line_dic['video_infor']
            task_lst.append(video_infor)
        for line in task_lst:
            url=line['url']
            try:
                video_dic=self.video_page(url)
                line['releaser']=video_dic['releaser']
                line['release_time']=video_dic['release_time']
                line['releaserUrl']=video_dic['releaserUrl']
                line['tags']=video_dic['tags']
                json_line=json.dumps(line)
                result_lst.append(line)
                result.write(json_line)
                result.write('\n')
                result.flush()
            except:
                print('no video infor')
        result.close()
        return result_lst

    def get_doc(self,cluename,taskname,resultname,donename):
        task=open(taskname)
        result=open(resultname,'a')
        clue=open(cluename,'a')
        done=open(donename,'a')
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls=line['url']
            urls=urls.replace('\n','')
            album_name=line['album_name']
            album_rating=line['album_rating']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = (" ".join(re.findall('showid: &#\d+;\d+',page))).split(';')[1]
                videoId = (" ".join(re.findall('videoId: &#\d+;\d+',page))).split(';')[1]
                videoCategoryId = (" ".join(re.findall('videoCategoryId: &#\d+;\d+',page))).split(';')[1]
                componentid = (" ".join(re.findall('"componentId":\d+',page))).split(':')[1]
                D0={'showid':showid ,'videoid':videoId,'videocategoryid':videoCategoryId,'componentid':componentid}
                json_D0 = json.dumps(D0)
                clue.write(json_D0)
                clue.write('\n')
                clue.flush()
                count=1
                thousand_10='万'
                url='http://v.youku.com/page/playlist?&l=debug&pm=3&vid='+videoId+'&fid=0&showid='+showid+'&sid=0&componentid='+componentid+'&videoCategoryId='+videoCategoryId+'&isSimple=false&page='+str(count)
                get_page = requests.get(url)
                get_page.encoding = 'utf-8'
                page = get_page.text
                page_dic = eval(page)
                video_infor = page_dic['html']
                soup=BeautifulSoup(video_infor,'html.parser')
                video_dic = soup.find_all('div',{'class':'item item-cover'})
                sumofvideo = len(video_dic)
                for line in video_dic:
                    title=line['title']
                    url = line.a['href']
                    if 'http:' not in url:
                        url='http:'+url
                    dura=line.find('span',{'class':'c-time'}).text
                    dura_lst=dura.split(':')
                    if len(dura_lst)==1:
                        duration=int(dura_lst[0])
                    elif len(dura_lst)==2:
                        duration=int(dura_lst[0])*60+int(dura_lst[1])
                    elif len(dura_lst)==3:
                        duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                    play_count_str=line.find('div',{'class':'status'}).text
                    play_count_str=play_count_str.replace('次播放','')
                    play_count_str=play_count_str.replace(',','')
                    if thousand_10 in play_count_str:
                        play_count_str=play_count_str.replace('万','')
                        play_count=int((float(play_count_str))*1e4)
                    else:
                        play_count=int(play_count_str)
                    fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                    D1={'album_name':album_name,'album_rating':album_rating,'title':title,
                        'duration':duration,'play_count':play_count,'url':url,'fetch_time':fetch_time}
                    if D1 not in result_lst:
                        result_lst.append(D1)
                        json_D1 = json.dumps(D1)
                        result.write(json_D1)
                        result.write('\n')
                        result.flush()
                        print(url)
                while sumofvideo == 100:
                    count+=1
                    url='http://v.youku.com/page/playlist?&l=debug&pm=3&vid='+videoId+'&fid=0&showid='+showid+'&sid=0&componentid='+componentid+'&videoCategoryId='+videoCategoryId+'&isSimple=false&page='+str(count)
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    page_dic = eval(page)
                    video_infor = page_dic['html']
                    if video_infor =="\n":
                        sumofvideo=0
                    else:
                        soup=BeautifulSoup(video_infor,'html.parser')
                        video_dic = soup.find_all('div',{'class':'item item-cover'})
                        sumofvideo = len(video_dic)
                        for line in video_dic:
                            title=line['title']
                            url = line.a['href']
                            if 'http:' not in url:
                                url='http:'+url
                            dura=line.find('span',{'class':'c-time'}).text
                            dura_lst=dura.split(':')
                            if len(dura_lst)==1:
                                duration=int(dura_lst[0])
                            elif len(dura_lst)==2:
                                duration=int(dura_lst[0])*60+int(dura_lst[1])
                            elif len(dura_lst)==3:
                                duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                            play_count_str=line.find('div',{'class':'status'}).text
                            play_count_str=play_count_str.replace('次播放','')
                            play_count_str=play_count_str.replace(',','')
                            if thousand_10 in play_count_str:
                                play_count_str=play_count_str.replace('万','')
                                play_count=int((float(play_count_str))*1e4)
                            else:
                                play_count=int(play_count_str)
                            fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                            D1={'album_name':album_name,'album_rating':album_rating,'title':title,
                                'duration':duration,'play_count':play_count,'url':url,'fetch_time':fetch_time}
                            if D1 not in result_lst:
                                result_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print(url)
                done.write(urls)
                done.write('\n')
                done.flush()
            except:
                pass
        done.close()
        task.close()
        result.close()
        clue.close()
        return result_lst


    def one_video(self,taskname):
        task=open(taskname)
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls=line['url']
            urls=urls.replace('\n','')
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = (" ".join(re.findall('showid: &#\d+;\d+',page))).split(';')[1]
                print('more video')
            except:
                result_lst.append(urls)
                print('one video')
        task.close()
        return result_lst



    def get_releaser(self,taskname,resultname,errorname):
        task=open(taskname)
        result=open(resultname,'a')
        error=open(errorname,'a')
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls=line['url']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            page2 = page.replace('\\','')
            try:
                releaser=re.findall("[\u4e00-\u9fa5]+","".join(re.findall('class="" title="[\u4e00-\u9fa5]+"',page2)))[0]
                print('get releaser')
            except:
                releaser=None
                error.write(urls)
                error.write('\n')
                error.flush()
                print('no releaser')
            line['releaser']=releaser
            result_lst.append(line)
            json_line=json.dumps(line)
            result.write(json_line)
            result.write('\n')
            result.flush()
        result.close()
        return result_lst


    def get_release_time(self,url):
        get_page=requests.get(url)
        page=get_page.text
        soup = BeautifulSoup(page,'html.parser')
        try:
            time_one=soup.find('span',{'class':'bold mr3'}).text
        except AttributeError:
            time_one=None
        try:
            time_two=time_one[4:]
        except  TypeError:
            time_two=None
        try:
            midsteptime=time_two.strip()
        except  AttributeError:
            midsteptime=None
        try:
            release_time = int(datetime.datetime.strptime(midsteptime, '%Y-%m-%d').timestamp()*1e3)
        except  (TypeError,ValueError):
            release_time = None
        D_video_page={"release_time":release_time,'url':url}
        print('get one line')
        return D_video_page


    def get_video_play_count_releaser_info(self,url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page,'html.parser')
        try:
            video_info = soup.find('div',{'class':'item item-cover current'})
            play_count_str = video_info.find('div',{'class':'status'}).text
            play_count_str = play_count_str.replace('次播放','')
            if '万' in play_count_str:
                play_count_str = play_count_str.replace('万','')
                play_count = int(float(play_count_str)*1e4)
            else:
                play_count = int(play_count_str.replace(',',''))
        except:
            play_count = 0
        try:
            releaser = soup.find('span',{'id':"module_basic_sub"}).a.text
            releaser = releaser.replace('\n','')
            releaser = releaser.replace(' ','')
            releaserUrl = soup.find('span',{'id':"module_basic_sub"}).a['href']
            if releaserUrl[0:4] == 'http':
                pass
            else:
                releaserUrl = 'http:' + releaserUrl
        except:
            releaser = None
            releaserUrl = None
        D0 = {'play_count':play_count, 'releaser':releaser, 'releaserUrl':releaserUrl}
        return D0


    def get_release_time_from_str(self,rt_str):
        minute = '分钟'
        hour = '小时'
        yesterday = '昨天'
        the_day_before_yesterday = '前天'
        the_day_before_many_days = '天前'
        if minute in rt_str or hour in rt_str :
            year = datetime.date.today().year
            month = datetime.date.today().month
            day = datetime.date.today().day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        elif yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days = 1)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        elif the_day_before_yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days = 2)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        elif the_day_before_many_days in rt_str:
            day_shift = int(re.findall('\d+',rt_str)[0])
            raw_data = datetime.date.today() - datetime.timedelta(days = day_shift)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        else:
            release_time = 0
        return release_time



    def list_page_all(self,
                      output_to_file=False, filepath=None,
                      output_to_es_raw=False,
                      output_to_es_register=False,
                      push_to_redis=False,
                      page_num_max=25):
        """
        Get all videos in youku.com '片库' in the home page bottom and the '全部'
        category with '最新发布' filter.
        At present (2018-06-28) the url is http://list.youku.com/category/video/c_0_d_1_s_2.html?spm=a2h1n.8251846.selectID.5!2~5~5!2~1~3!2~A
        This url changes to the following format
        'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str(count) + '.html'
        where count is page number.
        """
        # if page_num_max argument is passed in a value greater than 25, will use 25,
        # at present (2018-06-28) there is only 25 pages at most.
        if page_num_max > 25:
            page_num_max = 25
        result_Lst = []
        page_num = 1
        while page_num <= page_num_max:
            lst_url = 'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str(page_num) + '.html'
            print('Getting data on page %d: %s, %s'
                  % (page_num, lst_url, datetime.datetime.now()))
            get_page = retry_get_url(lst_url)
            if get_page is None:
                print('Failed to get page url: %s, will continue to next page.'
                      % lst_url)
                continue
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_agg:
                title = line.find('li', {'class': 'title'}).text
                dura_str = line.find('li', {'class': 'status'}).text
                dura_lst = dura_str.split(':')
                if len(dura_lst) == 3:
                    duration = int(int(dura_lst[0])*3600 + int(dura_lst[1])*60 + int(dura_lst[2]))
                elif len(dura_lst) == 2:
                    duration = int(int(dura_lst[0])*60 + int(dura_lst[1]))
                else:
                    duration = 0
                url = line.a['href']
                if url[0:4] != 'http':
                    url = 'https:' + url
                video_info_dic = self.get_video_play_count_releaser_info(url)
                fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)

                rt_str = line.find('li', {'class': ' '}).text
                release_time = self.get_release_time_from_str(rt_str)

                self.video_data['title'] = title
                self.video_data['duration'] = duration
                self.video_data['url'] = url
                self.video_data['play_count'] = video_info_dic['play_count']
                self.video_data['releaser'] = video_info_dic['releaser']
                self.video_data['releaserUrl'] = video_info_dic['releaserUrl']
                self.video_data['release_time'] = release_time
                self.video_data['fetch_time'] = fetch_time

                get_data = copy.deepcopy(self.video_data)
                result_Lst.append(get_data)

                if len(result_Lst) >= 100:
                    print('Got %d lines, %s' % (len(result_Lst),
                                                datetime.datetime.now()))
                    output_result(result_Lst, self.platform,
                                  output_to_file=output_to_file,
                                  filepath=filepath,
                                  output_to_es_raw=output_to_es_raw,
                                  output_to_es_register=output_to_es_register,
                                  push_to_redis=push_to_redis)
                    result_Lst.clear()

            page_num += 1

        if result_Lst != []:
            print('Got %d lines, %s' % (len(result_Lst),
                                        datetime.datetime.now()))
            output_result(result_Lst, self.platform,
                          output_to_file=output_to_file,
                          filepath=filepath,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          push_to_redis=push_to_redis)
            result_Lst.clear()

    def get_title_subtitle(self, url):
        get_page = retry_get_url(url)
        if get_page is None:
            print('Failed to get url: %s' % url)
            return
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        try:
            subtitle = soup.find('span', {'id': 'subtitle'}).text
        except:
            subtitle = None
        try:
            title = soup.h1.find('a', {'target':'_blank'}).text
            title_str = title + subtitle
        except:
            title_str = subtitle
        return title_str

    def select_tv_series_video(self, sp_video_album_name,
                               result_file=None, # modify to from es
                               output_to_file=False,
                               filepath=None,
                               output_to_es_raw=False,
                               output_to_es_register=False,
                               push_to_redis=False):
        select_result = []
        meta = Meta()
        result_lst = meta.dic_file_to_lst(result_file)
        for line in result_lst:
            url = line['url']
            title_str = self.get_title_subtitle(url)
            if sp_video_album_name in title_str:
                line.update({'title': title_str})
                select_result.append(line)
                print('it_belongs_to_what_we_want')
                if len(select_result) >= 100:
                    print('Got %d lines, %s' % (len(select_result), datetime.datetime.now()))
                    output_result(select_result, self.platform,
                                  output_to_file=output_to_file,
                                  filepath=filepath,
                                  output_to_es_raw=output_to_es_raw,
                                  output_to_es_register=output_to_es_register,
                                  push_to_redis=push_to_redis)
                    select_result.clear()
            else:
                print('not_belongs_to')
        if select_result != []:
            print('Got %d lines, %s' % (len(select_result),datetime.datetime.now()))
            output_result(select_result, self.platform,
                          output_to_file=output_to_file,
                          filepath=filepath,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          push_to_redis=push_to_redis)
            select_result.clear()



#list page asynchronous crawler
    def start_list_page(self, task_list,
                        output_to_file=False,
                        filepath=None,
                        output_to_es_raw=False,
                        push_to_redis=False,
                        es_index='crawler-data-raw',
                        doc_type='doc',
                        output_to_es_register=False,):
        self.list_page_task(task_list)
        self.download_list_page_single_process()


    def list_page_task(self, task_list, page_num_max=25):
        TASK_LIST = []
        for line in task_list:
            for num in range(1, page_num_max+1):
                url = line.replace('fangyucheng', str(num))
                TASK_LIST.append(url)
        random.shuffle(TASK_LIST)
        connect_with_redis.push_list_url_to_redis(platform=self.platform,
                                                  result_lst=TASK_LIST)

    async def download_page(self, session, url):
        get_page = await session.get(url)
        page = await get_page.text("utf-8", errors="ignore")
        return page

    async def download_list_page(self, loop):
        task_list = connect_with_redis.retrieve_list_url_from_redis(platform=self.platform)
        async with aiohttp.ClientSession() as session:
            task = [loop.create_task(self.download_page(session, url)) for url in task_list]
            done, pending = await asyncio.wait(task)
            RESULT_LIST = [d.result() for d in done]
            connect_with_redis.push_list_page_html_to_redis(platform=self.platform,
                                                            result_lst=RESULT_LIST)

    def download_list_page_single_process(self):
        key = 'youku_list_url'
        while connect_with_redis.length_of_lst(key) > 0:
            loop = asyncio.get_event_loop()
            loop.run_until_complete(self.download_list_page(loop))

    def parse_list_page_single_process(self):
        pid = os.getpid()
        URL_LIST = []
        key = 'youku_list_page_html'
        while connect_with_redis.length_of_lst(key) > 0:
            PAGE = connect_with_redis.retrieve_list_page_html_from_redis(platform=self.platform)
            soup = BeautifulSoup(PAGE, 'html.parser')
            video_url_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_url_agg:
                url = line.a['href']
                video_id_str = ' '.join(re.findall('id_.*html', url))
                video_id = video_id_str.replace('id_', '')
                tudou_url = 'https://video.tudou.com/v/%s' % video_id
                URL_LIST.append(tudou_url)
                connect_with_redis.push_video_url_to_redis_set(platform='new_tudou',
                                                               url_lst=URL_LIST)
            print("platform: %s, action: parse list page, process_id: %s, get %s urls"
                  % (self.platform, pid, len(URL_LIST)))

    def parse_list_page_multi_process(self, processes_num=20):
        pool = Pool(processes=processes_num)
        for num in range(processes_num):
            pool.apply_async(self.parse_list_page_single_process)
        pool.close()
        pool.join()


"""
due to the video play_count in new_tudou is equal to youku, and it is earier to get
play_count in new_tudou, I rebuild the youku video url so that I can use new_tudou
crawler to get data
"""
#    async def download_video_page(self, loop):
#        task_list = connect_with_redis.retrieve_video_url_from_redis_set(platform=self.platform)
#        async with aiohttp.ClientSession() as session:
#            task = [loop.create_task(self.download_page(session, url)) for url in task_list]
#            done, pending = await asyncio.wait(task)
#            RESULT_LIST = [d.result() for d in done]
#            connect_with_redis.push_video_page_html_to_redis(platform=self.platform,
#                                                             result_lst=RESULT_LIST)
#    def download_video_page_async_single_process(self):
#        key = 'youku_video_url'
#        pid = os.getpid()
#        while connect_with_redis.length_of_set(key) > 0:
#            loop = asyncio.get_event_loop()
#            loop.run_until_complete(self.download_video_page(loop))
#            print("platform: %s, action: download video page, process_id: %s"
#                  % (self.platform, pid))
#
#    def download_video_page_async_multi_process(self, processes_num=10):
#        pool = Pool(processes=processes_num)
#        for num in range(processes_num):
#            pool.apply_async(self.download_video_page_async_single_process)
#        pool.close()
#        pool.join()
#
#    def parse_video_page(self, resp_str):
#        soup = BeautifulSoup(resp_str, 'html.parser')
#        title = soup.find('meta', {'name': 'title'})['content']
#        url = soup.find('meta', {'itemprop': 'url'})['content']
#        release_time_str = soup.find('meta', {'itemprop': 'datePublished'})['content']
#        release_time = int(datetime.datetime.strptime(release_time_str,
#                                                      "%Y-%m-%d %H:%M:%S").timestamp()*1e3)
#        releaser = soup.find('span',{'id': "module_basic_sub"}).a.text
#        releaser = releaser.replace('\n', '')
#        releaser = releaser.replace(' ', '')
#        duration_str =  ' '.join(re.findall("seconds: '.*'", resp_str))
#        if '.' in duration_str:
#            duration_str = ' '.join(re.findall("seconds: '\d+.", duration_str))
#            duration = int(' '.join(re.findall('\d+', duration_str))) + 1
#        else:
#            duration = int(' '.join(re.findall('\d+', duration_str)))
#        releaserUrl = soup.find('span',{'id': "module_basic_sub"}).a['href']
#        if releaserUrl[0:4] != 'http':
#            releaserUrl = 'http:' + releaserUrl
#        try:
#            video_info = soup.find('div',{'class': 'item item-cover current'})
#            play_count_str = video_info.find('div',{'class': 'status'}).text
#            play_count = trans_play_count(play_count_str)
#        except:
#            print("can't find play count at %s" % url)
#            play_count = 0
#        try:
#            releaser = soup.find('span',{'id':"module_basic_sub"}).a.text
#            releaserUrl = soup.find('span',{'id':"module_basic_sub"}).a['href']
#        except:
#            releaser = None
#            releaserUrl = None
#        else:
#            releaser = releaser.replace('\n','')
#            releaser = releaser.replace(' ','')
#            if releaserUrl[0:4] != 'http':
#                releaserUrl = 'http:' + releaserUrl
#        fetch_time = int(time.time()*1e3)
#        video_info = {"title": title,
#                      'url': url,
#                      'duration': duration,
#                      'releaser': releaser,
#                      'play_count': play_count,
#                      'releaserUrl': releaserUrl,
#                      'release_time': release_time,
#                      'fetch_time': fetch_time}
#        return video_info
#
#    def parse_video_page_single_process(self, output_to_file=False,
#                                        filepath=None,
#                                        push_to_redis=False,
#                                        output_to_es_raw=False,
#                                        es_index=None,
#                                        doc_type=None,
#                                        output_to_es_register=False):
#        pid = os.getpid()
#        RESULT_LIST = []
#        key = 'youku_video_page_html'
#        while connect_with_redis.length_of_lst(key) > 0:
#            VIDEO_PAGE_HTML = connect_with_redis.retrieve_video_page_html_from_redis(platform=self.platform)
#            try:
#                video_info = self.parse_video_page(resp_str=VIDEO_PAGE_HTML)
#            except:
#                video_info = None
#                print("failed to parse video info")
#            if video_info is not None:
#                RESULT_LIST.append(video_info)
#                print("platform: %s, action: parse video page, process_id: %s, has done %s"
#                      % (self.platform, pid, len(RESULT_LIST)))
#                if len(RESULT_LIST) >= 100:
#                    output_result(result_Lst=RESULT_LIST,
#                                  platform=self.platform,
#                                  output_to_file=output_to_file,
#                                  filepath=filepath,
#                                  push_to_redis=push_to_redis,
#                                  output_to_es_raw=output_to_es_raw,
#                                  es_index=es_index,
#                                  doc_type=doc_type,
#                                  output_to_es_register=output_to_es_register)
#                    RESULT_LIST.clear()
#        if len(RESULT_LIST) != []:
#            output_result(result_Lst=RESULT_LIST,
#                          platform=self.platform,
#                          output_to_file=output_to_file,
#                          filepath=filepath,
#                          push_to_redis=push_to_redis,
#                          output_to_es_raw=output_to_es_raw,
#                          es_index=es_index,
#                          doc_type=doc_type,
#                          output_to_es_register=output_to_es_register)
#            RESULT_LIST.clear()
#
#    def parse_video_page_multi_process(self, para_dic,
#                                       processes_num=30):
#        pool = Pool(processes=processes_num)
#        for num in range(processes_num):
#            pool.apply_async(self.parse_video_page_single_process, kwds=para_dic)
#        pool.close()
#        pool.join()

# test
if __name__=='__main__':
    test = Crawler_youku()
    vid = 'XMjc5MTQyNjg0OA'
    test_selenium = test.get_video_play_count_by_vid(vid)
    sr_yk = test.search_page(keyword='任正非 BBC', search_pages_max=2)
