import json
import os
import string
from crawler import Crawler
from db.models import MemoriesMusics
from settings import Config


class DeezerCrawler(Crawler):
    REQUEST_DELAY = 0.11
    SERVICE = 'deezer'
    MODEL = MemoriesMusics

    @classmethod
    def api_url(cls, uid):
        return 'http://api.deezer.com/album/{uid}'.format(uid=uid)

    @classmethod
    def pretty_url(cls, uid):
        return 'http://deezer.com/album/{uid}'.format(uid=uid)

    @classmethod
    def urls(cls, _):
        uid = 100
        max_uid = 4000000
        while uid < max_uid:
            yield cls.api_url(uid)
            uid += 1

    @classmethod
    def transform(cls, content):
        Model = cls.MODEL
        response = json.loads(content)
        try:
            # if url is already registered then skip it
            Model.get(url=response['link'])
        except KeyError:
            return []
        except Model.DoesNotExist:
            return [cls.parse(response)]
        else:
            return []

    @classmethod
    def parse(cls, response):
        def summarize(response):
            genres = map(lambda genre: genre['name'],
                         response.get('genres', {}).get('data', []))
            return u'\n'.join(filter(None,
                                    [u'Genres: {}'.format(u', '.join(genres[:5])) if genres else None,
                                     u'#Tracks: {}'.format(response['nb_tracks']) if response.get('nb_tracks') else None,
                                     u'Label: {}'.format(response['label']) if response.get('label') else None]))

        year = lambda x: int(x) if x else None
        return {
            'title': response['title'],
            'year': year(response.get('release_date', '')[:4]),
            'url': response['link'],
            'summary': summarize(response),
            'artists': filter(None, [response.get('artist', {}).get('name', '')]),
        }

cf3090393ebf1bab842e10b0ce9239ad674aebc3
<mask token>


class DeezerCrawler(Crawler):
    <mask token>
    <mask token>
    <mask token>

    @classmethod
    def api_url(cls, uid):
        return 'http://api.deezer.com/album/{uid}'.format(uid=uid)
    <mask token>
    <mask token>

    @classmethod
    def transform(cls, content):
        Model = cls.MODEL
        response = json.loads(content)
        try:
            Model.get(url=response['link'])
        except KeyError:
            return []
        except Model.DoesNotExist:
            return [cls.parse(response)]
        else:
            return []

    @classmethod
    def parse(cls, response):

        def summarize(response):
            genres = map(lambda genre: genre['name'], response.get('genres',
                {}).get('data', []))
            return u'\n'.join(filter(None, [u'Genres: {}'.format(u', '.join
                (genres[:5])) if genres else None, u'#Tracks: {}'.format(
                response['nb_tracks']) if response.get('nb_tracks') else
                None, u'Label: {}'.format(response['label']) if response.
                get('label') else None]))
        year = lambda x: int(x) if x else None
        return {'title': response['title'], 'year': year(response.get(
            'release_date', '')[:4]), 'url': response['link'], 'summary':
            summarize(response), 'artists': filter(None, [response.get(
            'artist', {}).get('name', '')])}

<mask token>


class DeezerCrawler(Crawler):
    <mask token>
    <mask token>
    <mask token>

    @classmethod
    def api_url(cls, uid):
        return 'http://api.deezer.com/album/{uid}'.format(uid=uid)

    @classmethod
    def pretty_url(cls, uid):
        return 'http://deezer.com/album/{uid}'.format(uid=uid)

    @classmethod
    def urls(cls, _):
        uid = 100
        max_uid = 4000000
        while uid < max_uid:
            yield cls.api_url(uid)
            uid += 1

    @classmethod
    def transform(cls, content):
        Model = cls.MODEL
        response = json.loads(content)
        try:
            Model.get(url=response['link'])
        except KeyError:
            return []
        except Model.DoesNotExist:
            return [cls.parse(response)]
        else:
            return []

    @classmethod
    def parse(cls, response):

        def summarize(response):
            genres = map(lambda genre: genre['name'], response.get('genres',
                {}).get('data', []))
            return u'\n'.join(filter(None, [u'Genres: {}'.format(u', '.join
                (genres[:5])) if genres else None, u'#Tracks: {}'.format(
                response['nb_tracks']) if response.get('nb_tracks') else
                None, u'Label: {}'.format(response['label']) if response.
                get('label') else None]))
        year = lambda x: int(x) if x else None
        return {'title': response['title'], 'year': year(response.get(
            'release_date', '')[:4]), 'url': response['link'], 'summary':
            summarize(response), 'artists': filter(None, [response.get(
            'artist', {}).get('name', '')])}

<mask token>


class DeezerCrawler(Crawler):
    REQUEST_DELAY = 0.11
    SERVICE = 'deezer'
    MODEL = MemoriesMusics

    @classmethod
    def api_url(cls, uid):
        return 'http://api.deezer.com/album/{uid}'.format(uid=uid)

    @classmethod
    def pretty_url(cls, uid):
        return 'http://deezer.com/album/{uid}'.format(uid=uid)

    @classmethod
    def urls(cls, _):
        uid = 100
        max_uid = 4000000
        while uid < max_uid:
            yield cls.api_url(uid)
            uid += 1

    @classmethod
    def transform(cls, content):
        Model = cls.MODEL
        response = json.loads(content)
        try:
            Model.get(url=response['link'])
        except KeyError:
            return []
        except Model.DoesNotExist:
            return [cls.parse(response)]
        else:
            return []

    @classmethod
    def parse(cls, response):

        def summarize(response):
            genres = map(lambda genre: genre['name'], response.get('genres',
                {}).get('data', []))
            return u'\n'.join(filter(None, [u'Genres: {}'.format(u', '.join
                (genres[:5])) if genres else None, u'#Tracks: {}'.format(
                response['nb_tracks']) if response.get('nb_tracks') else
                None, u'Label: {}'.format(response['label']) if response.
                get('label') else None]))
        year = lambda x: int(x) if x else None
        return {'title': response['title'], 'year': year(response.get(
            'release_date', '')[:4]), 'url': response['link'], 'summary':
            summarize(response), 'artists': filter(None, [response.get(
            'artist', {}).get('name', '')])}

import json
import os
import string
from crawler import Crawler
from db.models import MemoriesMusics
from settings import Config


class DeezerCrawler(Crawler):
    REQUEST_DELAY = 0.11
    SERVICE = 'deezer'
    MODEL = MemoriesMusics

    @classmethod
    def api_url(cls, uid):
        return 'http://api.deezer.com/album/{uid}'.format(uid=uid)

    @classmethod
    def pretty_url(cls, uid):
        return 'http://deezer.com/album/{uid}'.format(uid=uid)

    @classmethod
    def urls(cls, _):
        uid = 100
        max_uid = 4000000
        while uid < max_uid:
            yield cls.api_url(uid)
            uid += 1

    @classmethod
    def transform(cls, content):
        Model = cls.MODEL
        response = json.loads(content)
        try:
            Model.get(url=response['link'])
        except KeyError:
            return []
        except Model.DoesNotExist:
            return [cls.parse(response)]
        else:
            return []

    @classmethod
    def parse(cls, response):

        def summarize(response):
            genres = map(lambda genre: genre['name'], response.get('genres',
                {}).get('data', []))
            return u'\n'.join(filter(None, [u'Genres: {}'.format(u', '.join
                (genres[:5])) if genres else None, u'#Tracks: {}'.format(
                response['nb_tracks']) if response.get('nb_tracks') else
                None, u'Label: {}'.format(response['label']) if response.
                get('label') else None]))
        year = lambda x: int(x) if x else None
        return {'title': response['title'], 'year': year(response.get(
            'release_date', '')[:4]), 'url': response['link'], 'summary':
            summarize(response), 'artists': filter(None, [response.get(
            'artist', {}).get('name', '')])}

import json
import os
import string
from crawler import Crawler
from db.models import MemoriesMusics
from settings import Config


class DeezerCrawler(Crawler):
    REQUEST_DELAY = 0.11
    SERVICE = 'deezer'
    MODEL = MemoriesMusics

    @classmethod
    def api_url(cls, uid):
        return 'http://api.deezer.com/album/{uid}'.format(uid=uid)

    @classmethod
    def pretty_url(cls, uid):
        return 'http://deezer.com/album/{uid}'.format(uid=uid)

    @classmethod
    def urls(cls, _):
        uid = 100
        max_uid = 4000000
        while uid < max_uid:
            yield cls.api_url(uid)
            uid += 1

    @classmethod
    def transform(cls, content):
        Model = cls.MODEL
        response = json.loads(content)
        try:
            # if url is already registered then skip it
            Model.get(url=response['link'])
        except KeyError:
            return []
        except Model.DoesNotExist:
            return [cls.parse(response)]
        else:
            return []

    @classmethod
    def parse(cls, response):
        def summarize(response):
            genres = map(lambda genre: genre['name'],
                         response.get('genres', {}).get('data', []))
            return u'\n'.join(filter(None,
                                    [u'Genres: {}'.format(u', '.join(genres[:5])) if genres else None,
                                     u'#Tracks: {}'.format(response['nb_tracks']) if response.get('nb_tracks') else None,
                                     u'Label: {}'.format(response['label']) if response.get('label') else None]))

        year = lambda x: int(x) if x else None
        return {
            'title': response['title'],
            'year': year(response.get('release_date', '')[:4]),
            'url': response['link'],
            'summary': summarize(response),
            'artists': filter(None, [response.get('artist', {}).get('name', '')]),
        }
