#encoding: utf-8
import requests
import time
import random
from threading import Thread
from queue import Queue
import pymongo

ua_list = [
  'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
  'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
  'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)',
]

citys = {'601': '鞍山', '825': '安顺', '723': '安阳', '671': '安庆', '862': '安康', '530': '北京', '570': '保定', '588': '包头', '717': '滨州', '856': '宝鸡', '678': '亳州', '789': '北海', '666': '蚌埠', '867': '白银', '620': '白城', '618': '白山', '817': '巴中', '828': '毕节', '613': '长春', '749': '长株潭', '801': '成都', '551': '重庆', '638': '常州', '573': '沧州', '755': '常德', '650': '常熟', '673': '滁州', '572': '承德', '758': '郴州', '781': '潮州', '579': '长治', '679': '池州', '600': '大连', '779': '东莞', '627': '大庆', '604': '丹东', '706': '东营', '715': '德州', '805': '德阳', '577': '大同', '815': '达州', '841': '大理', '592': '鄂尔多斯', '741': '鄂州', '748': '恩施', '681': '福州', '768': '佛山', '602': '抚顺', '700': '抚州', '674': '阜阳', '763': '广州', '822': '贵阳', '697': '赣州', '787': '桂林', '807': '广元', '814': '广安', '557': '甘肃', '622': '哈尔滨', '653': '杭州', '664': '合肥', '752': '衡阳', '575': '衡水', '568': '邯郸', '799': '海口', '587': '呼和浩特', '657': '湖州', '643': '淮安', '718': '菏泽', '669': '淮北', '860': '汉中', '593': '呼伦贝尔', '612': '葫芦岛', '773': '惠州', '724': '鹤壁', '667': '淮南', '10198': '珲春', '745': '黄冈', '737': '黄石', '672': '黄山', '760': '怀化', '632': '黑河', '776': '河源', '702': '济南', '614': '吉林', '656': '嘉兴', '769': '江门', '2512': '江阴', '694': '九江', '709': '济宁', '659': '金华', '744': '荆州', '726': '焦作', '782': '揭阳', '629': '佳木斯', '605': '锦州', '582': '晋中', '698': '吉安', '692': '景德镇', '580': '晋城', '10044': '济源', '404001': '吉首市', '720': '开封', '831': '昆明', '640': '昆山', '10118': '开平', '714': '临沂', '864': '兰州', '721': '洛阳', '642': '连云港', '716': '聊城', '786': '柳州', '574': '廊坊', '823': '六盘水', '729': '漯河', '713': '莱芜', '810': '乐山', '608': '辽阳', '689': '龙岩', '663': '丽水', '677': '六安', '585': '临汾', '586': '吕梁', '616': '辽源', '804': '泸州', '761': '娄底', '668': '马鞍山', '806': '绵阳', '774': '梅州', '771': '茂名', '631': '牡丹江', '811': '南充', '635': '南京', '731': '南阳', '654': '宁波', '691': '南昌', '641': '南通', '785': '南宁', '688': '南平', '690': '宁德', '809': '内江', '559': '宁夏', '722': '平顶山', '683': '莆田', '727': '濮阳', '3404': '普宁', '609': '盘锦', '803': '攀枝花', '693': '萍乡', '703': '青岛', '567': '秦皇岛', '685': '泉州', '778': '清远', '558': '青海', '832': '曲靖', '623': '齐齐哈尔', '660': '衢州', '827': '黔西南', '712': '日照', '538': '上海', '732': '商丘', '765': '深圳', '599': '沈阳', '565': '石家庄', '648': '宿迁', '639': '苏州', '675': '宿州', '747': '随州', '658': '绍兴', '800': '三亚', '767': '汕头', '615': '四平', '701': '上饶', '753': '邵阳', '738': '十堰', '730': '三门峡', '863': '商洛', '808': '遂宁', '684': '三明', '764': '韶关', '619': '松原', '633': '绥化', '775': '汕尾', '531': '天津', '576': '太原', '566': '唐山', '662': '台州', '610': '铁岭', '710': '泰安', '647': '泰州', '855': '铜川', '591': '通辽', '670': '铜陵', '617': '通化', '868': '天水', '826': '铜仁', '736': '武汉', '636': '无锡', '655': '温州', '711': '威海', '890': '乌鲁木齐', '665': '芜湖', '708': '潍坊', '858': '渭南', '854': '西安', '682': '厦门', '725': '新乡', '728': '许昌', '637': '徐州', '740': '襄阳', '555': '西藏', '857': '咸阳', '733': '信阳', '569': '邢台', '560': '新疆', '695': '新余', '743': '孝感', '680': '宣城', '746': '咸宁', '878': '西宁', '707': '烟台', '3376': '义乌', '606': '营口', '644': '盐城', '861': '榆林', '583': '运城', '739': '宜昌', '2513': '宜兴', '793': '玉林', '754': '岳阳', '813': '宜宾', '859': '延安', '621': '延边', '886': '银川', '578': '阳泉', '759': '永州', '777': '阳江', '757': '益阳', '699': '宜春', '833': '玉溪', '696': '鹰潭', '816': '雅安', '783': '云浮', '652': '张家港', '571': '张家口', '719': '郑州', '780': '中山', '766': '珠海', '704': '淄博', '687': '漳州', '646': '镇江', '770': '湛江', '772': '肇庆', '705': '枣庄', '824': '遵义', '735': '驻马店', '734': '周口', '802': '自贡', '661': '舟山', '870': '张掖', '756': '张家界'}
citys_codes = ['601', '825', '723', '671', '862', '530', '570', '588', '717', '856', '678', '789', '666', '867', '620', '618', '817', '828', '613', '749', '801', '551', '638', '573', '755', '650', '673', '572', '758', '781', '579', '679', '600', '779', '627', '604', '706', '715', '805', '577', '815', '841', '592', '741', '748', '681', '768', '602', '700', '674', '763', '822', '697', '787', '807', '814', '557', '622', '653', '664', '752', '575', '568', '799', '587', '657', '643', '718', '669', '860', '593', '612', '773', '724', '667', '10198', '745', '737', '672', '760', '632', '776', '702', '614', '656', '769', '2512', '694', '709', '659', '744', '726', '782', '629', '605', '582', '698', '692', '580', '10044', '404001', '720', '831', '640', '10118', '714', '864', '721', '642', '716', '786', '574', '823', '729', '713', '810', '608', '689', '663', '677', '585', '586', '616', '804', '761', '668', '806', '774', '771', '631', '811', '635', '731', '654', '691', '641', '785', '688', '690', '809', '559', '722', '683', '727', '3404', '609', '803', '693', '703', '567', '685', '778', '558', '832', '623', '660', '827', '712', '538', '732', '765', '599', '565', '648', '639', '675', '747', '658', '800', '767', '615', '701', '753', '738', '730', '863', '808', '684', '764', '619', '633', '775', '531', '576', '566', '662', '610', '710', '647', '855', '591', '670', '617', '868', '826', '736', '636', '655', '711', '890', '665', '708', '858', '854', '682', '725', '728', '637', '740', '555', '857', '733', '569', '560', '695', '743', '680', '746', '878', '707', '3376', '606', '644', '861', '583', '739', '2513', '793', '754', '813', '859', '621', '886', '578', '759', '777', '757', '699', '833', '696', '816', '783', '652', '571', '719', '780', '766', '704', '687', '646', '770', '772', '705', '824', '735', '734', '802', '661', '870', '756']

Top21_codes = ['736', '702', '854', '635', '765', '801', '763', '530', '600', '538', '749', '653', '531', '681', '664', '682', '639', '703', '599', '565', '551', '613']

class ZhiLianSpider(object):
    def __init__(self):
        self.one_url = 'https://fe-api.zhaopin.com/c/i/sou?start={}&pageSize=90&cityId={}&salary=0,0&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=java&kt=3'
        # self.two_url = ''
        self.q = Queue()

    # 获取响应内容函数
    def get_page(self, url):
        headers = {'User-Agent': random.choice(ua_list)}
        html = requests.get(url=url, headers=headers)
        html = html.json()

        return html

    # 主线函数: 获取所有数据
    def parse_page(self):
        while True:
            if not self.q.empty():
                one_url = self.q.get()
                try:
                    html = self.get_page(one_url)
                except:
                    break
                infolists = []
                for job in html['data']['results']:
                    jobName = job['jobName']  # 工作岗位
                    company = job['company']['name']  # 公司名称
                    type = job['company']['type']['name']  # 融资类型
                    try:
                        size = job['company']['size']['name']  # 公司规模
                    except KeyError as  e:
                        size = ''
                    city = job['city']['items'][0]['name']  # 城市
                    try:
                        city_name = job['city']['items'][1]['name']  # 地区
                    except:
                        city_name = city
                    updateDate = job['updateDate']  # 发布时间
                    salary = job['salary']  # 薪水
                    eduLevel = job['eduLevel']['name']  # 学历要求
                    try:
                        workingExp = job['workingExp']['name']  # 工作经验
                    except:
                        workingExp = '无经验'
                    positionURL = job['positionURL']  # 职位链接
                    welfare = job['welfare']  # 公司福利
                    item = {"工作岗位": jobName, "城市": city, '地区': city_name, "薪水": salary, "学历要求": eduLevel,
                            "工作经验": workingExp,
                            "公司名称": company, "融资类型": type, "公司规模": size, "发布时间": updateDate, "公司福利": welfare,
                            "职位链接": positionURL}

                    # 拼接二级地址,获取职责和要求
                    # two_url = self.two_url.format(positionURL)
                    # more_message = self.parse_two_page(two_url)
                    infolists.append(item)
                    print(item)
                # 每爬取按完成1页随机休眠
                collection.insert_many(infolists)
                time.sleep(random.uniform(0, 1))
            else:
                break

    # 解析二级页面函数
    # def parse_two_page(self, two_url):
    #     html = self.get_page(two_url)
    #     # 用replace处理一下特殊字符
    #     duty = html['Data']['Responsibility']
    #     duty = duty.replace('\r\n', '').replace('\n', '')
    #     # 处理要求
    #     require = html['Data']['Requirement']
    #     require = require.replace('\r\n', '').replace('\n', '')
    #
    #     return duty, require

    # 获取总页数
    def get_numbers(self,city_code):
        url = self.one_url.format(0,city_code)
        html = self.get_page(url)
        numbers = int(html['data']['count']) // 90 + 1

        return numbers

    def main(self,citycode):
        # one_url入队列
        number = self.get_numbers(citycode)
        for page in range(1, number + 1):
            one_url = self.one_url.format((page-1)*90,citycode)
            self.q.put(one_url)

        t_list = []
        for i in range(6):
            t = Thread(target=self.parse_page)
            t_list.append(t)
            t.start()

        for t in t_list:
            t.join()


if __name__ == '__main__':
    start = time.time()
    client = pymongo.MongoClient()  # 链接pymongo数据库
    db = client.DuoXianCheng  # 创建数据库
    collection = db.JiXian3
    spider = ZhiLianSpider()
    for citycode in Top21_codes[:5]:
        try:
            spider.main(citycode)
        except TypeError as e:
            print("该页面没有数据")
        except:
            print("页面请求失败")
            continue
    end = time.time()
    count = collection.find().count()
    print("获取到职位信息%s条" % count)
    print('执行时间:%.2f' % (end - start))


11d0d43bfaa6f0f7d7fcc76ab5db670413d79926
<mask token>


class ZhiLianSpider(object):
    <mask token>
    <mask token>

    def parse_page(self):
        while True:
            if not self.q.empty():
                one_url = self.q.get()
                try:
                    html = self.get_page(one_url)
                except:
                    break
                infolists = []
                for job in html['data']['results']:
                    jobName = job['jobName']
                    company = job['company']['name']
                    type = job['company']['type']['name']
                    try:
                        size = job['company']['size']['name']
                    except KeyError as e:
                        size = ''
                    city = job['city']['items'][0]['name']
                    try:
                        city_name = job['city']['items'][1]['name']
                    except:
                        city_name = city
                    updateDate = job['updateDate']
                    salary = job['salary']
                    eduLevel = job['eduLevel']['name']
                    try:
                        workingExp = job['workingExp']['name']
                    except:
                        workingExp = '无经验'
                    positionURL = job['positionURL']
                    welfare = job['welfare']
                    item = {'工作岗位': jobName, '城市': city, '地区': city_name,
                        '薪水': salary, '学历要求': eduLevel, '工作经验': workingExp,
                        '公司名称': company, '融资类型': type, '公司规模': size, '发布时间':
                        updateDate, '公司福利': welfare, '职位链接': positionURL}
                    infolists.append(item)
                    print(item)
                collection.insert_many(infolists)
                time.sleep(random.uniform(0, 1))
            else:
                break
    <mask token>

    def main(self, citycode):
        number = self.get_numbers(citycode)
        for page in range(1, number + 1):
            one_url = self.one_url.format((page - 1) * 90, citycode)
            self.q.put(one_url)
        t_list = []
        for i in range(6):
            t = Thread(target=self.parse_page)
            t_list.append(t)
            t.start()
        for t in t_list:
            t.join()


<mask token>

<mask token>


class ZhiLianSpider(object):
    <mask token>

    def get_page(self, url):
        headers = {'User-Agent': random.choice(ua_list)}
        html = requests.get(url=url, headers=headers)
        html = html.json()
        return html

    def parse_page(self):
        while True:
            if not self.q.empty():
                one_url = self.q.get()
                try:
                    html = self.get_page(one_url)
                except:
                    break
                infolists = []
                for job in html['data']['results']:
                    jobName = job['jobName']
                    company = job['company']['name']
                    type = job['company']['type']['name']
                    try:
                        size = job['company']['size']['name']
                    except KeyError as e:
                        size = ''
                    city = job['city']['items'][0]['name']
                    try:
                        city_name = job['city']['items'][1]['name']
                    except:
                        city_name = city
                    updateDate = job['updateDate']
                    salary = job['salary']
                    eduLevel = job['eduLevel']['name']
                    try:
                        workingExp = job['workingExp']['name']
                    except:
                        workingExp = '无经验'
                    positionURL = job['positionURL']
                    welfare = job['welfare']
                    item = {'工作岗位': jobName, '城市': city, '地区': city_name,
                        '薪水': salary, '学历要求': eduLevel, '工作经验': workingExp,
                        '公司名称': company, '融资类型': type, '公司规模': size, '发布时间':
                        updateDate, '公司福利': welfare, '职位链接': positionURL}
                    infolists.append(item)
                    print(item)
                collection.insert_many(infolists)
                time.sleep(random.uniform(0, 1))
            else:
                break

    def get_numbers(self, city_code):
        url = self.one_url.format(0, city_code)
        html = self.get_page(url)
        numbers = int(html['data']['count']) // 90 + 1
        return numbers

    def main(self, citycode):
        number = self.get_numbers(citycode)
        for page in range(1, number + 1):
            one_url = self.one_url.format((page - 1) * 90, citycode)
            self.q.put(one_url)
        t_list = []
        for i in range(6):
            t = Thread(target=self.parse_page)
            t_list.append(t)
            t.start()
        for t in t_list:
            t.join()


<mask token>

<mask token>


class ZhiLianSpider(object):

    def __init__(self):
        self.one_url = (
            'https://fe-api.zhaopin.com/c/i/sou?start={}&pageSize=90&cityId={}&salary=0,0&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=java&kt=3'
            )
        self.q = Queue()

    def get_page(self, url):
        headers = {'User-Agent': random.choice(ua_list)}
        html = requests.get(url=url, headers=headers)
        html = html.json()
        return html

    def parse_page(self):
        while True:
            if not self.q.empty():
                one_url = self.q.get()
                try:
                    html = self.get_page(one_url)
                except:
                    break
                infolists = []
                for job in html['data']['results']:
                    jobName = job['jobName']
                    company = job['company']['name']
                    type = job['company']['type']['name']
                    try:
                        size = job['company']['size']['name']
                    except KeyError as e:
                        size = ''
                    city = job['city']['items'][0]['name']
                    try:
                        city_name = job['city']['items'][1]['name']
                    except:
                        city_name = city
                    updateDate = job['updateDate']
                    salary = job['salary']
                    eduLevel = job['eduLevel']['name']
                    try:
                        workingExp = job['workingExp']['name']
                    except:
                        workingExp = '无经验'
                    positionURL = job['positionURL']
                    welfare = job['welfare']
                    item = {'工作岗位': jobName, '城市': city, '地区': city_name,
                        '薪水': salary, '学历要求': eduLevel, '工作经验': workingExp,
                        '公司名称': company, '融资类型': type, '公司规模': size, '发布时间':
                        updateDate, '公司福利': welfare, '职位链接': positionURL}
                    infolists.append(item)
                    print(item)
                collection.insert_many(infolists)
                time.sleep(random.uniform(0, 1))
            else:
                break

    def get_numbers(self, city_code):
        url = self.one_url.format(0, city_code)
        html = self.get_page(url)
        numbers = int(html['data']['count']) // 90 + 1
        return numbers

    def main(self, citycode):
        number = self.get_numbers(citycode)
        for page in range(1, number + 1):
            one_url = self.one_url.format((page - 1) * 90, citycode)
            self.q.put(one_url)
        t_list = []
        for i in range(6):
            t = Thread(target=self.parse_page)
            t_list.append(t)
            t.start()
        for t in t_list:
            t.join()


if __name__ == '__main__':
    start = time.time()
    client = pymongo.MongoClient()
    db = client.DuoXianCheng
    collection = db.JiXian3
    spider = ZhiLianSpider()
    for citycode in Top21_codes[:5]:
        try:
            spider.main(citycode)
        except TypeError as e:
            print('该页面没有数据')
        except:
            print('页面请求失败')
            continue
    end = time.time()
    count = collection.find().count()
    print('获取到职位信息%s条' % count)
    print('执行时间:%.2f' % (end - start))

<mask token>
ua_list = [
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1'
    ,
    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
    'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)'
    ]
citys = {'601': '鞍山', '825': '安顺', '723': '安阳', '671': '安庆', '862': '安康',
    '530': '北京', '570': '保定', '588': '包头', '717': '滨州', '856': '宝鸡', '678':
    '亳州', '789': '北海', '666': '蚌埠', '867': '白银', '620': '白城', '618': '白山',
    '817': '巴中', '828': '毕节', '613': '长春', '749': '长株潭', '801': '成都', '551':
    '重庆', '638': '常州', '573': '沧州', '755': '常德', '650': '常熟', '673': '滁州',
    '572': '承德', '758': '郴州', '781': '潮州', '579': '长治', '679': '池州', '600':
    '大连', '779': '东莞', '627': '大庆', '604': '丹东', '706': '东营', '715': '德州',
    '805': '德阳', '577': '大同', '815': '达州', '841': '大理', '592': '鄂尔多斯',
    '741': '鄂州', '748': '恩施', '681': '福州', '768': '佛山', '602': '抚顺', '700':
    '抚州', '674': '阜阳', '763': '广州', '822': '贵阳', '697': '赣州', '787': '桂林',
    '807': '广元', '814': '广安', '557': '甘肃', '622': '哈尔滨', '653': '杭州', '664':
    '合肥', '752': '衡阳', '575': '衡水', '568': '邯郸', '799': '海口', '587': '呼和浩特',
    '657': '湖州', '643': '淮安', '718': '菏泽', '669': '淮北', '860': '汉中', '593':
    '呼伦贝尔', '612': '葫芦岛', '773': '惠州', '724': '鹤壁', '667': '淮南', '10198':
    '珲春', '745': '黄冈', '737': '黄石', '672': '黄山', '760': '怀化', '632': '黑河',
    '776': '河源', '702': '济南', '614': '吉林', '656': '嘉兴', '769': '江门', '2512':
    '江阴', '694': '九江', '709': '济宁', '659': '金华', '744': '荆州', '726': '焦作',
    '782': '揭阳', '629': '佳木斯', '605': '锦州', '582': '晋中', '698': '吉安', '692':
    '景德镇', '580': '晋城', '10044': '济源', '404001': '吉首市', '720': '开封', '831':
    '昆明', '640': '昆山', '10118': '开平', '714': '临沂', '864': '兰州', '721': '洛阳',
    '642': '连云港', '716': '聊城', '786': '柳州', '574': '廊坊', '823': '六盘水',
    '729': '漯河', '713': '莱芜', '810': '乐山', '608': '辽阳', '689': '龙岩', '663':
    '丽水', '677': '六安', '585': '临汾', '586': '吕梁', '616': '辽源', '804': '泸州',
    '761': '娄底', '668': '马鞍山', '806': '绵阳', '774': '梅州', '771': '茂名', '631':
    '牡丹江', '811': '南充', '635': '南京', '731': '南阳', '654': '宁波', '691': '南昌',
    '641': '南通', '785': '南宁', '688': '南平', '690': '宁德', '809': '内江', '559':
    '宁夏', '722': '平顶山', '683': '莆田', '727': '濮阳', '3404': '普宁', '609': '盘锦',
    '803': '攀枝花', '693': '萍乡', '703': '青岛', '567': '秦皇岛', '685': '泉州',
    '778': '清远', '558': '青海', '832': '曲靖', '623': '齐齐哈尔', '660': '衢州',
    '827': '黔西南', '712': '日照', '538': '上海', '732': '商丘', '765': '深圳', '599':
    '沈阳', '565': '石家庄', '648': '宿迁', '639': '苏州', '675': '宿州', '747': '随州',
    '658': '绍兴', '800': '三亚', '767': '汕头', '615': '四平', '701': '上饶', '753':
    '邵阳', '738': '十堰', '730': '三门峡', '863': '商洛', '808': '遂宁', '684': '三明',
    '764': '韶关', '619': '松原', '633': '绥化', '775': '汕尾', '531': '天津', '576':
    '太原', '566': '唐山', '662': '台州', '610': '铁岭', '710': '泰安', '647': '泰州',
    '855': '铜川', '591': '通辽', '670': '铜陵', '617': '通化', '868': '天水', '826':
    '铜仁', '736': '武汉', '636': '无锡', '655': '温州', '711': '威海', '890': '乌鲁木齐',
    '665': '芜湖', '708': '潍坊', '858': '渭南', '854': '西安', '682': '厦门', '725':
    '新乡', '728': '许昌', '637': '徐州', '740': '襄阳', '555': '西藏', '857': '咸阳',
    '733': '信阳', '569': '邢台', '560': '新疆', '695': '新余', '743': '孝感', '680':
    '宣城', '746': '咸宁', '878': '西宁', '707': '烟台', '3376': '义乌', '606': '营口',
    '644': '盐城', '861': '榆林', '583': '运城', '739': '宜昌', '2513': '宜兴', '793':
    '玉林', '754': '岳阳', '813': '宜宾', '859': '延安', '621': '延边', '886': '银川',
    '578': '阳泉', '759': '永州', '777': '阳江', '757': '益阳', '699': '宜春', '833':
    '玉溪', '696': '鹰潭', '816': '雅安', '783': '云浮', '652': '张家港', '571': '张家口',
    '719': '郑州', '780': '中山', '766': '珠海', '704': '淄博', '687': '漳州', '646':
    '镇江', '770': '湛江', '772': '肇庆', '705': '枣庄', '824': '遵义', '735': '驻马店',
    '734': '周口', '802': '自贡', '661': '舟山', '870': '张掖', '756': '张家界'}
citys_codes = ['601', '825', '723', '671', '862', '530', '570', '588',
    '717', '856', '678', '789', '666', '867', '620', '618', '817', '828',
    '613', '749', '801', '551', '638', '573', '755', '650', '673', '572',
    '758', '781', '579', '679', '600', '779', '627', '604', '706', '715',
    '805', '577', '815', '841', '592', '741', '748', '681', '768', '602',
    '700', '674', '763', '822', '697', '787', '807', '814', '557', '622',
    '653', '664', '752', '575', '568', '799', '587', '657', '643', '718',
    '669', '860', '593', '612', '773', '724', '667', '10198', '745', '737',
    '672', '760', '632', '776', '702', '614', '656', '769', '2512', '694',
    '709', '659', '744', '726', '782', '629', '605', '582', '698', '692',
    '580', '10044', '404001', '720', '831', '640', '10118', '714', '864',
    '721', '642', '716', '786', '574', '823', '729', '713', '810', '608',
    '689', '663', '677', '585', '586', '616', '804', '761', '668', '806',
    '774', '771', '631', '811', '635', '731', '654', '691', '641', '785',
    '688', '690', '809', '559', '722', '683', '727', '3404', '609', '803',
    '693', '703', '567', '685', '778', '558', '832', '623', '660', '827',
    '712', '538', '732', '765', '599', '565', '648', '639', '675', '747',
    '658', '800', '767', '615', '701', '753', '738', '730', '863', '808',
    '684', '764', '619', '633', '775', '531', '576', '566', '662', '610',
    '710', '647', '855', '591', '670', '617', '868', '826', '736', '636',
    '655', '711', '890', '665', '708', '858', '854', '682', '725', '728',
    '637', '740', '555', '857', '733', '569', '560', '695', '743', '680',
    '746', '878', '707', '3376', '606', '644', '861', '583', '739', '2513',
    '793', '754', '813', '859', '621', '886', '578', '759', '777', '757',
    '699', '833', '696', '816', '783', '652', '571', '719', '780', '766',
    '704', '687', '646', '770', '772', '705', '824', '735', '734', '802',
    '661', '870', '756']
Top21_codes = ['736', '702', '854', '635', '765', '801', '763', '530',
    '600', '538', '749', '653', '531', '681', '664', '682', '639', '703',
    '599', '565', '551', '613']


class ZhiLianSpider(object):

    def __init__(self):
        self.one_url = (
            'https://fe-api.zhaopin.com/c/i/sou?start={}&pageSize=90&cityId={}&salary=0,0&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=java&kt=3'
            )
        self.q = Queue()

    def get_page(self, url):
        headers = {'User-Agent': random.choice(ua_list)}
        html = requests.get(url=url, headers=headers)
        html = html.json()
        return html

    def parse_page(self):
        while True:
            if not self.q.empty():
                one_url = self.q.get()
                try:
                    html = self.get_page(one_url)
                except:
                    break
                infolists = []
                for job in html['data']['results']:
                    jobName = job['jobName']
                    company = job['company']['name']
                    type = job['company']['type']['name']
                    try:
                        size = job['company']['size']['name']
                    except KeyError as e:
                        size = ''
                    city = job['city']['items'][0]['name']
                    try:
                        city_name = job['city']['items'][1]['name']
                    except:
                        city_name = city
                    updateDate = job['updateDate']
                    salary = job['salary']
                    eduLevel = job['eduLevel']['name']
                    try:
                        workingExp = job['workingExp']['name']
                    except:
                        workingExp = '无经验'
                    positionURL = job['positionURL']
                    welfare = job['welfare']
                    item = {'工作岗位': jobName, '城市': city, '地区': city_name,
                        '薪水': salary, '学历要求': eduLevel, '工作经验': workingExp,
                        '公司名称': company, '融资类型': type, '公司规模': size, '发布时间':
                        updateDate, '公司福利': welfare, '职位链接': positionURL}
                    infolists.append(item)
                    print(item)
                collection.insert_many(infolists)
                time.sleep(random.uniform(0, 1))
            else:
                break

    def get_numbers(self, city_code):
        url = self.one_url.format(0, city_code)
        html = self.get_page(url)
        numbers = int(html['data']['count']) // 90 + 1
        return numbers

    def main(self, citycode):
        number = self.get_numbers(citycode)
        for page in range(1, number + 1):
            one_url = self.one_url.format((page - 1) * 90, citycode)
            self.q.put(one_url)
        t_list = []
        for i in range(6):
            t = Thread(target=self.parse_page)
            t_list.append(t)
            t.start()
        for t in t_list:
            t.join()


if __name__ == '__main__':
    start = time.time()
    client = pymongo.MongoClient()
    db = client.DuoXianCheng
    collection = db.JiXian3
    spider = ZhiLianSpider()
    for citycode in Top21_codes[:5]:
        try:
            spider.main(citycode)
        except TypeError as e:
            print('该页面没有数据')
        except:
            print('页面请求失败')
            continue
    end = time.time()
    count = collection.find().count()
    print('获取到职位信息%s条' % count)
    print('执行时间:%.2f' % (end - start))

#encoding: utf-8
import requests
import time
import random
from threading import Thread
from queue import Queue
import pymongo

ua_list = [
  'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
  'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
  'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)',
]

citys = {'601': '鞍山', '825': '安顺', '723': '安阳', '671': '安庆', '862': '安康', '530': '北京', '570': '保定', '588': '包头', '717': '滨州', '856': '宝鸡', '678': '亳州', '789': '北海', '666': '蚌埠', '867': '白银', '620': '白城', '618': '白山', '817': '巴中', '828': '毕节', '613': '长春', '749': '长株潭', '801': '成都', '551': '重庆', '638': '常州', '573': '沧州', '755': '常德', '650': '常熟', '673': '滁州', '572': '承德', '758': '郴州', '781': '潮州', '579': '长治', '679': '池州', '600': '大连', '779': '东莞', '627': '大庆', '604': '丹东', '706': '东营', '715': '德州', '805': '德阳', '577': '大同', '815': '达州', '841': '大理', '592': '鄂尔多斯', '741': '鄂州', '748': '恩施', '681': '福州', '768': '佛山', '602': '抚顺', '700': '抚州', '674': '阜阳', '763': '广州', '822': '贵阳', '697': '赣州', '787': '桂林', '807': '广元', '814': '广安', '557': '甘肃', '622': '哈尔滨', '653': '杭州', '664': '合肥', '752': '衡阳', '575': '衡水', '568': '邯郸', '799': '海口', '587': '呼和浩特', '657': '湖州', '643': '淮安', '718': '菏泽', '669': '淮北', '860': '汉中', '593': '呼伦贝尔', '612': '葫芦岛', '773': '惠州', '724': '鹤壁', '667': '淮南', '10198': '珲春', '745': '黄冈', '737': '黄石', '672': '黄山', '760': '怀化', '632': '黑河', '776': '河源', '702': '济南', '614': '吉林', '656': '嘉兴', '769': '江门', '2512': '江阴', '694': '九江', '709': '济宁', '659': '金华', '744': '荆州', '726': '焦作', '782': '揭阳', '629': '佳木斯', '605': '锦州', '582': '晋中', '698': '吉安', '692': '景德镇', '580': '晋城', '10044': '济源', '404001': '吉首市', '720': '开封', '831': '昆明', '640': '昆山', '10118': '开平', '714': '临沂', '864': '兰州', '721': '洛阳', '642': '连云港', '716': '聊城', '786': '柳州', '574': '廊坊', '823': '六盘水', '729': '漯河', '713': '莱芜', '810': '乐山', '608': '辽阳', '689': '龙岩', '663': '丽水', '677': '六安', '585': '临汾', '586': '吕梁', '616': '辽源', '804': '泸州', '761': '娄底', '668': '马鞍山', '806': '绵阳', '774': '梅州', '771': '茂名', '631': '牡丹江', '811': '南充', '635': '南京', '731': '南阳', '654': '宁波', '691': '南昌', '641': '南通', '785': '南宁', '688': '南平', '690': '宁德', '809': '内江', '559': '宁夏', '722': '平顶山', '683': '莆田', '727': '濮阳', '3404': '普宁', '609': '盘锦', '803': '攀枝花', '693': '萍乡', '703': '青岛', '567': '秦皇岛', '685': '泉州', '778': '清远', '558': '青海', '832': '曲靖', '623': '齐齐哈尔', '660': '衢州', '827': '黔西南', '712': '日照', '538': '上海', '732': '商丘', '765': '深圳', '599': '沈阳', '565': '石家庄', '648': '宿迁', '639': '苏州', '675': '宿州', '747': '随州', '658': '绍兴', '800': '三亚', '767': '汕头', '615': '四平', '701': '上饶', '753': '邵阳', '738': '十堰', '730': '三门峡', '863': '商洛', '808': '遂宁', '684': '三明', '764': '韶关', '619': '松原', '633': '绥化', '775': '汕尾', '531': '天津', '576': '太原', '566': '唐山', '662': '台州', '610': '铁岭', '710': '泰安', '647': '泰州', '855': '铜川', '591': '通辽', '670': '铜陵', '617': '通化', '868': '天水', '826': '铜仁', '736': '武汉', '636': '无锡', '655': '温州', '711': '威海', '890': '乌鲁木齐', '665': '芜湖', '708': '潍坊', '858': '渭南', '854': '西安', '682': '厦门', '725': '新乡', '728': '许昌', '637': '徐州', '740': '襄阳', '555': '西藏', '857': '咸阳', '733': '信阳', '569': '邢台', '560': '新疆', '695': '新余', '743': '孝感', '680': '宣城', '746': '咸宁', '878': '西宁', '707': '烟台', '3376': '义乌', '606': '营口', '644': '盐城', '861': '榆林', '583': '运城', '739': '宜昌', '2513': '宜兴', '793': '玉林', '754': '岳阳', '813': '宜宾', '859': '延安', '621': '延边', '886': '银川', '578': '阳泉', '759': '永州', '777': '阳江', '757': '益阳', '699': '宜春', '833': '玉溪', '696': '鹰潭', '816': '雅安', '783': '云浮', '652': '张家港', '571': '张家口', '719': '郑州', '780': '中山', '766': '珠海', '704': '淄博', '687': '漳州', '646': '镇江', '770': '湛江', '772': '肇庆', '705': '枣庄', '824': '遵义', '735': '驻马店', '734': '周口', '802': '自贡', '661': '舟山', '870': '张掖', '756': '张家界'}
citys_codes = ['601', '825', '723', '671', '862', '530', '570', '588', '717', '856', '678', '789', '666', '867', '620', '618', '817', '828', '613', '749', '801', '551', '638', '573', '755', '650', '673', '572', '758', '781', '579', '679', '600', '779', '627', '604', '706', '715', '805', '577', '815', '841', '592', '741', '748', '681', '768', '602', '700', '674', '763', '822', '697', '787', '807', '814', '557', '622', '653', '664', '752', '575', '568', '799', '587', '657', '643', '718', '669', '860', '593', '612', '773', '724', '667', '10198', '745', '737', '672', '760', '632', '776', '702', '614', '656', '769', '2512', '694', '709', '659', '744', '726', '782', '629', '605', '582', '698', '692', '580', '10044', '404001', '720', '831', '640', '10118', '714', '864', '721', '642', '716', '786', '574', '823', '729', '713', '810', '608', '689', '663', '677', '585', '586', '616', '804', '761', '668', '806', '774', '771', '631', '811', '635', '731', '654', '691', '641', '785', '688', '690', '809', '559', '722', '683', '727', '3404', '609', '803', '693', '703', '567', '685', '778', '558', '832', '623', '660', '827', '712', '538', '732', '765', '599', '565', '648', '639', '675', '747', '658', '800', '767', '615', '701', '753', '738', '730', '863', '808', '684', '764', '619', '633', '775', '531', '576', '566', '662', '610', '710', '647', '855', '591', '670', '617', '868', '826', '736', '636', '655', '711', '890', '665', '708', '858', '854', '682', '725', '728', '637', '740', '555', '857', '733', '569', '560', '695', '743', '680', '746', '878', '707', '3376', '606', '644', '861', '583', '739', '2513', '793', '754', '813', '859', '621', '886', '578', '759', '777', '757', '699', '833', '696', '816', '783', '652', '571', '719', '780', '766', '704', '687', '646', '770', '772', '705', '824', '735', '734', '802', '661', '870', '756']

Top21_codes = ['736', '702', '854', '635', '765', '801', '763', '530', '600', '538', '749', '653', '531', '681', '664', '682', '639', '703', '599', '565', '551', '613']

class ZhiLianSpider(object):
    def __init__(self):
        self.one_url = 'https://fe-api.zhaopin.com/c/i/sou?start={}&pageSize=90&cityId={}&salary=0,0&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=java&kt=3'
        # self.two_url = ''
        self.q = Queue()

    # 获取响应内容函数
    def get_page(self, url):
        headers = {'User-Agent': random.choice(ua_list)}
        html = requests.get(url=url, headers=headers)
        html = html.json()

        return html

    # 主线函数: 获取所有数据
    def parse_page(self):
        while True:
            if not self.q.empty():
                one_url = self.q.get()
                try:
                    html = self.get_page(one_url)
                except:
                    break
                infolists = []
                for job in html['data']['results']:
                    jobName = job['jobName']  # 工作岗位
                    company = job['company']['name']  # 公司名称
                    type = job['company']['type']['name']  # 融资类型
                    try:
                        size = job['company']['size']['name']  # 公司规模
                    except KeyError as  e:
                        size = ''
                    city = job['city']['items'][0]['name']  # 城市
                    try:
                        city_name = job['city']['items'][1]['name']  # 地区
                    except:
                        city_name = city
                    updateDate = job['updateDate']  # 发布时间
                    salary = job['salary']  # 薪水
                    eduLevel = job['eduLevel']['name']  # 学历要求
                    try:
                        workingExp = job['workingExp']['name']  # 工作经验
                    except:
                        workingExp = '无经验'
                    positionURL = job['positionURL']  # 职位链接
                    welfare = job['welfare']  # 公司福利
                    item = {"工作岗位": jobName, "城市": city, '地区': city_name, "薪水": salary, "学历要求": eduLevel,
                            "工作经验": workingExp,
                            "公司名称": company, "融资类型": type, "公司规模": size, "发布时间": updateDate, "公司福利": welfare,
                            "职位链接": positionURL}

                    # 拼接二级地址,获取职责和要求
                    # two_url = self.two_url.format(positionURL)
                    # more_message = self.parse_two_page(two_url)
                    infolists.append(item)
                    print(item)
                # 每爬取按完成1页随机休眠
                collection.insert_many(infolists)
                time.sleep(random.uniform(0, 1))
            else:
                break

    # 解析二级页面函数
    # def parse_two_page(self, two_url):
    #     html = self.get_page(two_url)
    #     # 用replace处理一下特殊字符
    #     duty = html['Data']['Responsibility']
    #     duty = duty.replace('\r\n', '').replace('\n', '')
    #     # 处理要求
    #     require = html['Data']['Requirement']
    #     require = require.replace('\r\n', '').replace('\n', '')
    #
    #     return duty, require

    # 获取总页数
    def get_numbers(self,city_code):
        url = self.one_url.format(0,city_code)
        html = self.get_page(url)
        numbers = int(html['data']['count']) // 90 + 1

        return numbers

    def main(self,citycode):
        # one_url入队列
        number = self.get_numbers(citycode)
        for page in range(1, number + 1):
            one_url = self.one_url.format((page-1)*90,citycode)
            self.q.put(one_url)

        t_list = []
        for i in range(6):
            t = Thread(target=self.parse_page)
            t_list.append(t)
            t.start()

        for t in t_list:
            t.join()


if __name__ == '__main__':
    start = time.time()
    client = pymongo.MongoClient()  # 链接pymongo数据库
    db = client.DuoXianCheng  # 创建数据库
    collection = db.JiXian3
    spider = ZhiLianSpider()
    for citycode in Top21_codes[:5]:
        try:
            spider.main(citycode)
        except TypeError as e:
            print("该页面没有数据")
        except:
            print("页面请求失败")
            continue
    end = time.time()
    count = collection.find().count()
    print("获取到职位信息%s条" % count)
    print('执行时间:%.2f' % (end - start))

