#!/usr/local/bin/python
# encoding: utf-8
import re
import os
import json, pprint
from urllib2 import Request, urlopen, URLError, HTTPError
import time
import sys
from pymongo import Connection, DESCENDING
import datetime

db_conn=""
collectid=0

LOG_LEVEL_VERBOS = 0
LOG_LEVEL_DEBUG = 1
LOG_LEVEL_INFO = 2
LOG_LEVEL_WARN = 3
LOG_LEVEL_ERROR = 4
LOG_LEVEL = LOG_LEVEL_VERBOS
DB_GATE = True
MaxPage=0  #0 actual

###########################################################33
CATEGROY_MOVIE = 1
CATEGROY_TVSERIES = 2
CATEGROY_CARTOON = 4
CATEGROY_VARIETY = 6

db_conn = Connection(host="localhost", port=27017)

coll_album_raw = db_conn['videos_raw2'].albums
coll_detail_raw = db_conn['videos_raw2'].details
coll_detail_raw.ensure_index('updateat')

coll_album = db_conn['videos'].album6
coll_album.ensure_index('id', unique=True)
coll_album.ensure_index('name')
coll_album.ensure_index('cat')
coll_album.ensure_index('hhot')
coll_album.ensure_index('hnew')

#coll_detail = db_conn['videos'].detail6
#coll_detail.ensure_index('id', unique=True)

success_cnt = 0
fail_cnt = 0

def get_album_info(src,dt):
    item = {}

    #print "dt:",dt
    item['id'] = 'vd0_' + src['id'] #vendor0:vst
    item['name'] = src['title']
    #item['alias'] = dt['']
    item['cat'] = dt['type']
    if src.has_key('qxd'):
        item['qxd'] = src['qxd']
    #if dt['url'] is None:
    #    print "None:",dt['did']
    #    return

    #if not dt['url'].has_key('site'):
    #    print 'None url:',dt['url']
    #    return

    site = dt['playlist']
    item['sites'] = site
    item['area'] = dt['area'].split("/")
    item['type'] = dt['cate'].split("/")
    item['lang'] = dt['language']
    item['img'] = src['img']
    if src.has_key('mark'):
        item['score'] = src['mark']
    item['scov'] = dt['banben'] #sub cover
    #item['dur'] = dt['mlong']
    item['mtype'] = 0 #dt['contentType']
    #item['def'] = 2 if dt['isSD']==1 else 0

    #item['vendor'] = "iqiyi"
    #item['tv'] = dt['']
    #item['tags'] = dt['leafctgs'].split(" ")[:-1]

    rdate = {}
    rdate["year"] = dt['year']
    item['rdate'] = rdate

    directors = dt['director'].split("/")
    actors = dt['actor'].split("/")
    roles = {}
    roles["dirs"] = directors
    roles["acts"] = actors
    #roles.voic = src['']
    item['roles'] = roles

    item['cdate'] = datetime.datetime.now()

    item['paid'] = 0 #src['isPurchase']
    item['recd'] = 0
    if dt.has_key('new_top'):
        related = dt['new_top'];
        for r in related:
            r['id'] = 'vd0_' + r['id']
            r['score'] = r['mark']
            r['name'] = r['title']
            del r['mark']
            if r.has_key('idesc'):
                del r['idesc']
            del r['title']
        item['relat'] = related
    item['del'] = 0
    item['hhot'] = 100000 - src['ord']

    #if item['cat'] == CATEGROY_TVSERIES:
    #    item['sets'] = src['sets'] #todo
    #    item['cset'] = src['sets'] #todo
    item['desc'] = dt['info']

    return item

def get_video_info_variety(**dt):
    subset = {}
    subset['album'] = dt['source']
    subset['name'] = dt['albumName']
    subset['subtit'] = dt['subTitle']
    subset['subset'] = dt['tvYear']
    subset['dur'] = dt['timeLength']
    subset['img'] = dt['picUrl']
    subset['desc'] = dt['desc']
    subset['ptype'] = 0
    subset['url'] = dt['playUrl']
    subset['purl'] = dt['playUrl0']
    return subset

def process(dt):
    src = coll_album_raw.find_one({"id":dt['id']})
    if src is None:
        print "album is none! ", dt['title'], " ", dt['id']
        return

    item = get_album_info(src, dt)
    if item is None:
        return

    #print "update item:", item['id']
    doc = coll_album.find_one({'id':item['id']},{"id":1})
    if doc != None:
        item['_id'] = doc['_id']
    #print item
    coll_album.save(item)

def getDatetimeFromStr(s):
    format = '%Y-%m-%d %H:%M:%S.%f'
    return datetime.datetime.strptime(s, format)

if __name__=='__main__':
    print "main start"
    d = datetime.datetime.now() - datetime.timedelta(days=1) #from 1 day ago
    cursor = coll_detail_raw.find({"updateat": {"$gt": d}}) #{"name":"刀客家族的女人"})
    #cursor = coll_detail_raw.find()
    totalToPcd = cursor.count()
    print str(datetime.datetime.now()) + " totalToProcess:",totalToPcd
    i = 0;
    for item in cursor:
        print 'progress: %d/%d %d%%\r' % (i,totalToPcd, (i*100/totalToPcd)),;
        if item['playlist'].__len__() > 0:
            process(item)
            i+=1;

9983fc4fb21fd796cb5f1d234b53734be6f635a6
#!/usr/local/bin/python
# encoding: utf-8
import re
import os
import json, pprint
from urllib2 import Request, urlopen, URLError, HTTPError
import time
import sys
from pymongo import Connection, DESCENDING
import datetime

db_conn=""
collectid=0

LOG_LEVEL_VERBOS = 0
LOG_LEVEL_DEBUG = 1
LOG_LEVEL_INFO = 2
LOG_LEVEL_WARN = 3
LOG_LEVEL_ERROR = 4
LOG_LEVEL = LOG_LEVEL_VERBOS
DB_GATE = True
MaxPage=0  #0 actual

###########################################################33
CATEGROY_MOVIE = 1
CATEGROY_TVSERIES = 2
CATEGROY_CARTOON = 4
CATEGROY_VARIETY = 6

db_conn = Connection(host="localhost", port=27017)

coll_album_raw = db_conn['videos_raw2'].albums
coll_detail_raw = db_conn['videos_raw2'].details
coll_detail_raw.ensure_index('updateat')

coll_album = db_conn['videos'].album6
coll_album.ensure_index('id', unique=True)
coll_album.ensure_index('name')
coll_album.ensure_index('cat')
coll_album.ensure_index('hhot')
coll_album.ensure_index('hnew')

#coll_detail = db_conn['videos'].detail6
#coll_detail.ensure_index('id', unique=True)

success_cnt = 0
fail_cnt = 0

def get_album_info(src,dt):
    item = {}

    #print "dt:",dt
    item['id'] = 'vd0_' + src['id'] #vendor0:vst
    item['name'] = src['title']
    #item['alias'] = dt['']
    item['cat'] = dt['type']
    if src.has_key('qxd'):
        item['qxd'] = src['qxd']
    #if dt['url'] is None:
    #    print "None:",dt['did']
    #    return

    #if not dt['url'].has_key('site'):
    #    print 'None url:',dt['url']
    #    return

    site = dt['playlist']
    item['sites'] = site
    item['area'] = dt['area'].split("/")
    item['type'] = dt['cate'].split("/")
    item['lang'] = dt['language']
    item['img'] = src['img']
    if src.has_key('mark'):
        item['score'] = src['mark']
    item['scov'] = dt['banben'] #sub cover
    #item['dur'] = dt['mlong']
    item['mtype'] = 0 #dt['contentType']
    #item['def'] = 2 if dt['isSD']==1 else 0

    #item['vendor'] = "iqiyi"
    #item['tv'] = dt['']
    #item['tags'] = dt['leafctgs'].split(" ")[:-1]

    rdate = {}
    rdate["year"] = dt['year']
    item['rdate'] = rdate

    directors = dt['director'].split("/")
    actors = dt['actor'].split("/")
    roles = {}
    roles["dirs"] = directors
    roles["acts"] = actors
    #roles.voic = src['']
    item['roles'] = roles

    item['cdate'] = datetime.datetime.now()

    item['paid'] = 0 #src['isPurchase']
    item['recd'] = 0
    if dt.has_key('new_top'):
        related = dt['new_top'];
        for r in related:
            r['id'] = 'vd0_' + r['id']
            r['score'] = r['mark']
            r['name'] = r['title']
            del r['mark']
            if r.has_key('idesc'):
                del r['idesc']
            del r['title']
        item['relat'] = related
    item['del'] = 0
    item['hhot'] = 100000 - src['ord']

    #if item['cat'] == CATEGROY_TVSERIES:
    #    item['sets'] = src['sets'] #todo
    #    item['cset'] = src['sets'] #todo
    item['desc'] = dt['info']

    return item

def get_video_info_variety(**dt):
    subset = {}
    subset['album'] = dt['source']
    subset['name'] = dt['albumName']
    subset['subtit'] = dt['subTitle']
    subset['subset'] = dt['tvYear']
    subset['dur'] = dt['timeLength']
    subset['img'] = dt['picUrl']
    subset['desc'] = dt['desc']
    subset['ptype'] = 0
    subset['url'] = dt['playUrl']
    subset['purl'] = dt['playUrl0']
    return subset

def process(dt):
    src = coll_album_raw.find_one({"id":dt['id']})
    if src is None:
        print "album is none! ", dt['title'], " ", dt['id']
        return

    item = get_album_info(src, dt)
    if item is None:
        return

    #print "update item:", item['id']
    doc = coll_album.find_one({'id':item['id']},{"id":1})
    if doc != None:
        item['_id'] = doc['_id']
    #print item
    coll_album.save(item)

def getDatetimeFromStr(s):
    format = '%Y-%m-%d %H:%M:%S.%f'
    return datetime.datetime.strptime(s, format)

if __name__=='__main__':
    print "main start"
    d = datetime.datetime.now() - datetime.timedelta(days=1) #from 1 day ago
    cursor = coll_detail_raw.find({"updateat": {"$gt": d}}) #{"name":"刀客家族的女人"})
    #cursor = coll_detail_raw.find()
    totalToPcd = cursor.count()
    print str(datetime.datetime.now()) + " totalToProcess:",totalToPcd
    i = 0;
    for item in cursor:
        print 'progress: %d/%d %d%%\r' % (i,totalToPcd, (i*100/totalToPcd)),;
        if item['playlist'].__len__() > 0:
            process(item)
            i+=1;
