code:
import collections
import itertools
import json
import random
import re

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
    compat_str,
    compat_urllib_parse_urlencode,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    UserNotLive,
    base_url,
    clean_html,
    dict_get,
    float_or_none,
    int_or_none,
    make_archive_id,
    parse_duration,
    parse_iso8601,
    parse_qs,
    qualities,
    str_or_none,
    traverse_obj,
    try_get,
    unified_timestamp,
    update_url_query,
    url_or_none,
    urljoin,
)

class TwitchBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:(?:www|go|m)\.)?twitch\.tv'

    _API_BASE = 'https://api.twitch.tv'
    _USHER_BASE = 'https://usher.ttvnw.net'
    _LOGIN_FORM_URL = 'https://www.twitch.tv/login'
    _LOGIN_POST_URL = 'https://passport.twitch.tv/login'
    _CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
    _NETRC_MACHINE = 'twitch'

    _OPERATION_HASHES = {
        'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
        'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
        'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
        'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
        'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
        'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
        'VideoAccessToken_Clip': '36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11',
        'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
        'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
        'VideoPlayer_ChapterSelectButtonVideo': '8d2793384aac3773beab5e59bd5d6f585aedb923d292800119e03d40cd0f9b41',
        'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
    }

    def _perform_login(self, username, password):
        def fail(message):
            raise ExtractorError(
                'Unable to login. Twitch said: %s' % message, expected=True)

        def login_step(page, urlh, note, data):
            form = self._hidden_inputs(page)
            form.update(data)

            page_url = urlh.geturl()
            post_url = self._search_regex(
                r'<form[^>]+action=(["\'])(?P<url>.+?)\1', page,
                'post url', default=self._LOGIN_POST_URL, group='url')
            post_url = urljoin(page_url, post_url)

            headers = {
                'Referer': page_url,
                'Origin': 'https://www.twitch.tv',
                'Content-Type': 'text/plain;charset=UTF-8',
            }

            response = self._download_json(
                post_url, None, note, data=json.dumps(form).encode(),
                headers=headers, expected_status=400)
            error = dict_get(response, ('error', 'error_description', 'error_code'))
            if error:
                fail(error)

            if 'Authenticated successfully' in response.get('message', ''):
                return None, None

            redirect_url = urljoin(
                post_url,
                response.get('redirect') or response['redirect_path'])
            return self._download_webpage_handle(
                redirect_url, None, 'Downloading login redirect page',
                headers=headers)

        login_page, handle = self._download_webpage_handle(
            self._LOGIN_FORM_URL, None, 'Downloading login page')

        # Some TOR nodes and public proxies are blocked completely
        if 'blacklist_message' in login_page:
            fail(clean_html(login_page))

        redirect_page, handle = login_step(
            login_page, handle, 'Logging in', {
                'username': username,
                'password': password,
                'client_id': self._CLIENT_ID,
            })

        # Successful login
        if not redirect_page:
            return

        if re.search(r'(?i)<form[^>]+id="two-factor-submit"', redirect_page) is not None:
            # TODO: Add mechanism to request an SMS or phone call
            tfa_token = self._get_tfa_info('two-factor authentication token')
            login_step(redirect_page, handle, 'Submitting TFA token', {
                'authy_token': tfa_token,
                'remember_2fa': 'true',
            })

    def _prefer_source(self, formats):
        try:
            source = next(f for f in formats if f['format_id'] == 'Source')
            source['quality'] = 10
        except StopIteration:
            for f in formats:
                if '/chunked/' in f['url']:
                    f.update({
                        'quality': 10,
                        'format_note': 'Source',
                    })
        self._sort_formats(formats)

    def _download_base_gql(self, video_id, ops, note, fatal=True):
        headers = {
            'Content-Type': 'text/plain;charset=UTF-8',
            'Client-ID': self._CLIENT_ID,
        }
        gql_auth = self._get_cookies('https://gql.twitch.tv').get('auth-token')
        if gql_auth:
            headers['Authorization'] = 'OAuth ' + gql_auth.value
        return self._download_json(
            'https://gql.twitch.tv/gql', video_id, note,
            data=json.dumps(ops).encode(),
            headers=headers, fatal=fatal)

    def _download_gql(self, video_id, ops, note, fatal=True):
        for op in ops:
            op['extensions'] = {
                'persistedQuery': {
                    'version': 1,
                    'sha256Hash': self._OPERATION_HASHES[op['operationName']],
                }
            }
        return self._download_base_gql(video_id, ops, note)

    def _download_access_token(self, video_id, token_kind, param_name):
        method = '%sPlaybackAccessToken' % token_kind
        ops = {
            'query': '''{
              %s(
                %s: "%s",
                params: {
                  platform: "web",
                  playerBackend: "mediaplayer",
                  playerType: "site"
                }
              )
              {
                value
                signature
              }
            }''' % (method, param_name, video_id),
        }
        return self._download_base_gql(
            video_id, ops,
            'Downloading %s access token GraphQL' % token_kind)['data'][method]

class TwitchVodIE(TwitchBaseIE):
    IE_NAME = 'twitch:vod'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/v(?:ideo)?|videos)/|
                            player\.twitch\.tv/\?.*?\bvideo=v?
                        )
                        (?P<id>\d+)
                    '''

    _TESTS = [{
        'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
        'info_dict': {
            'id': 'v6528877',
            'ext': 'mp4',
            'title': 'LCK Summer Split - Week 6 Day 1',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 17208,
            'timestamp': 1435131734,
            'upload_date': '20150624',
            'uploader': 'Riot Games',
            'uploader_id': 'riotgames',
            'view_count': int,
            'start_time': 310,
            'chapters': [
                {
                    'start_time': 0,
                    'end_time': 17208,
                    'title': 'League of Legends'
                }
            ],
            'live_status': 'was_live',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # Untitled broadcast (title is None)
        'url': 'http://www.twitch.tv/belkao_o/v/11230755',
        'info_dict': {
            'id': 'v11230755',
            'ext': 'mp4',
            'title': 'Untitled Broadcast',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 1638,
            'timestamp': 1439746708,
            'upload_date': '20150816',
            'uploader': 'BelkAO_o',
            'uploader_id': 'belkao_o',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'skip': 'HTTP Error 404: Not Found',
    }, {
        'url': 'http://player.twitch.tv/?t=5m10s&video=v6528877',
        'only_matching': True,
    }, {
        'url': 'https://www.twitch.tv/videos/6528877',
        'only_matching': True,
    }, {
        'url': 'https://m.twitch.tv/beagsandjam/v/247478721',
        'only_matching': True,
    }, {
        'url': 'https://www.twitch.tv/northernlion/video/291940395',
        'only_matching': True,
    }, {
        'url': 'https://player.twitch.tv/?video=480452374',
        'only_matching': True,
    }, {
        'url': 'https://www.twitch.tv/videos/635475444',
        'info_dict': {
            'id': 'v635475444',
            'ext': 'mp4',
            'title': 'Riot Games',
            'duration': 11643,
            'uploader': 'Riot Games',
            'uploader_id': 'riotgames',
            'timestamp': 1590770569,
            'upload_date': '20200529',
            'chapters': [
                {
                    'start_time': 0,
                    'end_time': 573,
                    'title': 'League of Legends'
                },
                {
                    'start_time': 573,
                    'end_time': 3922,
                    'title': 'Legends of Runeterra'
                },
                {
                    'start_time': 3922,
                    'end_time': 11643,
                    'title': 'Art'
                }
            ],
            'live_status': 'was_live',
            'thumbnail': r're:^https?://.*\.jpg$',
            'view_count': int,
        },
        'params': {
            'skip_download': True
        },
    }, {
        'note': 'Storyboards',
        'url': 'https://www.twitch.tv/videos/635475444',
        'info_dict': {
            'id': 'v635475444',
            'format_id': 'sb0',
            'ext': 'mhtml',
            'title': 'Riot Games',
            'duration': 11643,
            'uploader': 'Riot Games',
            'uploader_id': 'riotgames',
            'timestamp': 1590770569,
            'upload_date': '20200529',
            'chapters': [
                {
                    'start_time': 0,
                    'end_time': 573,
                    'title': 'League of Legends'
                },
                {
                    'start_time': 573,
                    'end_time': 3922,
                    'title': 'Legends of Runeterra'
                },
                {
                    'start_time': 3922,
                    'end_time': 11643,
                    'title': 'Art'
                }
            ],
            'live_status': 'was_live',
            'thumbnail': r're:^https?://.*\.jpg$',
            'view_count': int,
            'columns': int,
            'rows': int,
        },
        'params': {
            'format': 'mhtml',
            'skip_download': True
        }
    }, {
        'note': 'VOD with single chapter',
        'url': 'https://www.twitch.tv/videos/1536751224',
        'info_dict': {
            'id': 'v1536751224',
            'ext': 'mp4',
            'title': 'Porter Robinson Star Guardian Stream Tour with LilyPichu',
            'duration': 8353,
            'uploader': 'Riot Games',
            'uploader_id': 'riotgames',
            'timestamp': 1658267731,
            'upload_date': '20220719',
            'chapters': [
                {
                    'start_time': 0,
                    'end_time': 8353,
                    'title': 'League of Legends'
                }
            ],
            'live_status': 'was_live',
            'thumbnail': r're:^https?://.*\.jpg$',
            'view_count': int,
        },
        'params': {
            'skip_download': True
        },
        'expected_warnings': ['Unable to download JSON metadata: HTTP Error 403: Forbidden']
    }]

    def _download_info(self, item_id):
        data = self._download_gql(
            item_id, [{
                'operationName': 'VideoMetadata',
                'variables': {
                    'channelLogin': '',
                    'videoID': item_id,
                },
            }, {
                'operationName': 'VideoPlayer_ChapterSelectButtonVideo',
                'variables': {
                    'includePrivate': False,
                    'videoID': item_id,
                },
            }, {
                'operationName': 'VideoPlayer_VODSeekbarPreviewVideo',
                'variables': {
                    'includePrivate': False,
                    'videoID': item_id,
                },
            }],
            'Downloading stream metadata GraphQL')

        video = traverse_obj(data, (0, 'data', 'video'))
        video['moments'] = traverse_obj(data, (1, 'data', 'video', 'moments', 'edges', ..., 'node'))
        video['storyboard'] = traverse_obj(data, (2, 'data', 'video', 'seekPreviewsURL'), expected_type=url_or_none)

        if video is None:
            raise ExtractorError(
                'Video %s does not exist' % item_id, expected=True)
        return video

    def _extract_info(self, info):
        status = info.get('status')
        if status == 'recording':
            is_live = True
        elif status == 'recorded':
            is_live = False
        else:
            is_live = None
        _QUALITIES = ('small', 'medium', 'large')
        quality_key = qualities(_QUALITIES)
        thumbnails = []
        preview = info.get('preview')
        if isinstance(preview, dict):
            for thumbnail_id, thumbnail_url in preview.items():
                thumbnail_url = url_or_none(thumbnail_url)
                if not thumbnail_url:
                    continue
                if thumbnail_id not in _QUALITIES:
                    continue
                thumbnails.append({
                    'url': thumbnail_url,
                    'preference': quality_key(thumbnail_id),
                })
        return {
            'id': info['_id'],
            'title': info.get('title') or 'Untitled Broadcast',
            'description': info.get('description'),
            'duration': int_or_none(info.get('length')),
            'thumbnails': thumbnails,
            'uploader': info.get('channel', {}).get('display_name'),
            'uploader_id': info.get('channel', {}).get('name'),
            'timestamp': parse_iso8601(info.get('recorded_at')),
            'view_count': int_or_none(info.get('views')),
            'is_live': is_live,
            'was_live': True,
        }

    def _extract_chapters(self, info, item_id):
        if not info.get('moments'):
            game = traverse_obj(info, ('game', 'displayName'))
            if game:
                yield {'title': game}
            return

        for moment in info['moments']:
            start_time = int_or_none(moment.get('positionMilliseconds'), 1000)
            duration = int_or_none(moment.get('durationMilliseconds'), 1000)
            name = str_or_none(moment.get('description'))

            if start_time is None or duration is None:
                self.report_warning(f'Important chapter information missing for chapter {name}', item_id)
                continue
            yield {
                'start_time': start_time,
                'end_time': start_time + duration,
                'title': name,
            }

    def _extract_info_gql(self, info, item_id):
        vod_id = info.get('id') or item_id
        # id backward compatibility for download archives
        if vod_id[0] != 'v':
            vod_id = 'v%s' % vod_id
        thumbnail = url_or_none(info.get('previewThumbnailURL'))
        is_live = None
        if thumbnail:
            if thumbnail.endswith('/404_processing_{width}x{height}.png'):
                is_live, thumbnail = True, None
            else:
                is_live = False
                for p in ('width', 'height'):
                    thumbnail = thumbnail.replace('{%s}' % p, '0')

        return {
            'id': vod_id,
            'title': info.get('title') or 'Untitled Broadcast',
            'description': info.get('description'),
            'duration': int_or_none(info.get('lengthSeconds')),
            'thumbnail': thumbnail,
            'uploader': try_get(info, lambda x: x['owner']['displayName'], compat_str),
            'uploader_id': try_get(info, lambda x: x['owner']['login'], compat_str),
            'timestamp': unified_timestamp(info.get('publishedAt')),
            'view_count': int_or_none(info.get('viewCount')),
            'chapters': list(self._extract_chapters(info, item_id)),
            'is_live': is_live,
            'was_live': True,
        }

    def _extract_storyboard(self, item_id, storyboard_json_url, duration):
        if not duration or not storyboard_json_url:
            return
        spec = self._download_json(storyboard_json_url, item_id, 'Downloading storyboard metadata JSON', fatal=False) or []
        # sort from highest quality to lowest
        # This makes sb0 the highest-quality format, sb1 - lower, etc which is consistent with youtube sb ordering
        spec.sort(key=lambda x: int_or_none(x.get('width')) or 0, reverse=True)
        base = base_url(storyboard_json_url)
        for i, s in enumerate(spec):
            count = int_or_none(s.get('count'))
            images = s.get('images')
            if not (images and count):
                continue
            fragment_duration = duration / len(images)
            yield {
                'format_id': f'sb{i}',
                'format_note': 'storyboard',
                'ext': 'mhtml',
                'protocol': 'mhtml',
                'acodec': 'none',
                'vcodec': 'none',
                'url': urljoin(base, images[0]),
                'width': int_or_none(s.get('width')),
                'height': int_or_none(s.get('height')),
                'fps': count / duration,
                'rows': int_or_none(s.get('rows')),
                'columns': int_or_none(s.get('cols')),
                'fragments': [{
                    'url': urljoin(base, path),
                    'duration': fragment_duration,
                } for path in images],
            }

    def _real_extract(self, url):
        vod_id = self._match_id(url)

        video = self._download_info(vod_id)
        info = self._extract_info_gql(video, vod_id)
        access_token = self._download_access_token(vod_id, 'video', 'id')

        formats = self._extract_m3u8_formats(
            '%s/vod/%s.m3u8?%s' % (
                self._USHER_BASE, vod_id,
                compat_urllib_parse_urlencode({
                    'allow_source': 'true',
                    'allow_audio_only': 'true',
                    'allow_spectre': 'true',
                    'player': 'twitchweb',
                    'playlist_include_framerate': 'true',
                    'nauth': access_token['value'],
                    'nauthsig': access_token['signature'],
                })),
            vod_id, 'mp4', entry_protocol='m3u8_native')

        formats.extend(self._extract_storyboard(vod_id, video.get('storyboard'), info.get('duration')))

        self._prefer_source(formats)
        info['formats'] = formats

        parsed_url = compat_urllib_parse_urlparse(url)
        query = compat_parse_qs(parsed_url.query)
        if 't' in query:
            info['start_time'] = parse_duration(query['t'][0])

        if info.get('timestamp') is not None:
            info['subtitles'] = {
                'rechat': [{
                    'url': update_url_query(
                        'https://api.twitch.tv/v5/videos/%s/comments' % vod_id, {
                            'client_id': self._CLIENT_ID,
                        }),
                    'ext': 'json',
                }],
            }

        return info

def _make_video_result(node):
    assert isinstance(node, dict)
    video_id = node.get('id')
    if not video_id:
        return
    return {
        '_type': 'url_transparent',
        'ie_key': TwitchVodIE.ie_key(),
        'id': 'v' + video_id,
        'url': 'https://www.twitch.tv/videos/%s' % video_id,
        'title': node.get('title'),
        'thumbnail': node.get('previewThumbnailURL'),
        'duration': float_or_none(node.get('lengthSeconds')),
        'view_count': int_or_none(node.get('viewCount')),
    }

class TwitchCollectionIE(TwitchBaseIE):
    _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/collections/(?P<id>[^/]+)'

    _TESTS = [{
        'url': 'https://www.twitch.tv/collections/wlDCoH0zEBZZbQ',
        'info_dict': {
            'id': 'wlDCoH0zEBZZbQ',
            'title': 'Overthrow Nook, capitalism for children',
        },
        'playlist_mincount': 13,
    }]

    _OPERATION_NAME = 'CollectionSideBar'

    def _real_extract(self, url):
        collection_id = self._match_id(url)
        collection = self._download_gql(
            collection_id, [{
                'operationName': self._OPERATION_NAME,
                'variables': {'collectionID': collection_id},
            }],
            'Downloading collection GraphQL')[0]['data']['collection']
        title = collection.get('title')
        entries = []
        for edge in collection['items']['edges']:
            if not isinstance(edge, dict):
                continue
            node = edge.get('node')
            if not isinstance(node, dict):
                continue
            video = _make_video_result(node)
            if video:
                entries.append(video)
        return self.playlist_result(
            entries, playlist_id=collection_id, playlist_title=title)

class TwitchPlaylistBaseIE(TwitchBaseIE):
    _PAGE_LIMIT = 100

    def _entries(self, channel_name, *args):
        cursor = None
        variables_common = self._make_variables(channel_name, *args)
        entries_key = '%ss' % self._ENTRY_KIND
        for page_num in itertools.count(1):
            variables = variables_common.copy()
            variables['limit'] = self._PAGE_LIMIT
            if cursor:
                variables['cursor'] = cursor
            page = self._download_gql(
                channel_name, [{
                    'operationName': self._OPERATION_NAME,
                    'variables': variables,
                }],
                'Downloading %ss GraphQL page %s' % (self._NODE_KIND, page_num),
                fatal=False)
            if not page:
                break
            edges = try_get(
                page, lambda x: x[0]['data']['user'][entries_key]['edges'], list)
            if not edges:
                break
            for edge in edges:
                if not isinstance(edge, dict):
                    continue
                if edge.get('__typename') != self._EDGE_KIND:
                    continue
                node = edge.get('node')
                if not isinstance(node, dict):
                    continue
                if node.get('__typename') != self._NODE_KIND:
                    continue
                entry = self._extract_entry(node)
                if entry:
                    cursor = edge.get('cursor')
                    yield entry
            if not cursor or not isinstance(cursor, compat_str):
                break

class TwitchVideosIE(TwitchPlaylistBaseIE):
    _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:videos|profile)'

    _TESTS = [{
        # All Videos sorted by Date
        'url': 'https://www.twitch.tv/spamfish/videos?filter=all',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - All Videos sorted by Date',
        },
        'playlist_mincount': 924,
    }, {
        # All Videos sorted by Popular
        'url': 'https://www.twitch.tv/spamfish/videos?filter=all&sort=views',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - All Videos sorted by Popular',
        },
        'playlist_mincount': 931,
    }, {
        # Past Broadcasts sorted by Date
        'url': 'https://www.twitch.tv/spamfish/videos?filter=archives',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - Past Broadcasts sorted by Date',
        },
        'playlist_mincount': 27,
    }, {
        # Highlights sorted by Date
        'url': 'https://www.twitch.tv/spamfish/videos?filter=highlights',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - Highlights sorted by Date',
        },
        'playlist_mincount': 901,
    }, {
        # Uploads sorted by Date
        'url': 'https://www.twitch.tv/esl_csgo/videos?filter=uploads&sort=time',
        'info_dict': {
            'id': 'esl_csgo',
            'title': 'esl_csgo - Uploads sorted by Date',
        },
        'playlist_mincount': 5,
    }, {
        # Past Premieres sorted by Date
        'url': 'https://www.twitch.tv/spamfish/videos?filter=past_premieres',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - Past Premieres sorted by Date',
        },
        'playlist_mincount': 1,
    }, {
        'url': 'https://www.twitch.tv/spamfish/videos/all',
        'only_matching': True,
    }, {
        'url': 'https://m.twitch.tv/spamfish/videos/all',
        'only_matching': True,
    }, {
        'url': 'https://www.twitch.tv/spamfish/videos',
        'only_matching': True,
    }]

    Broadcast = collections.namedtuple('Broadcast', ['type', 'label'])

    _DEFAULT_BROADCAST = Broadcast(None, 'All Videos')
    _BROADCASTS = {
        'archives': Broadcast('ARCHIVE', 'Past Broadcasts'),
        'highlights': Broadcast('HIGHLIGHT', 'Highlights'),
        'uploads': Broadcast('UPLOAD', 'Uploads'),
        'past_premieres': Broadcast('PAST_PREMIERE', 'Past Premieres'),
        'all': _DEFAULT_BROADCAST,
    }

    _DEFAULT_SORTED_BY = 'Date'
    _SORTED_BY = {
        'time': _DEFAULT_SORTED_BY,
        'views': 'Popular',
    }

    _OPERATION_NAME = 'FilterableVideoTower_Videos'
    _ENTRY_KIND = 'video'
    _EDGE_KIND = 'VideoEdge'
    _NODE_KIND = 'Video'

    @classmethod
    def suitable(cls, url):
        return (False
                if any(ie.suitable(url) for ie in (
                    TwitchVideosClipsIE,
                    TwitchVideosCollectionsIE))
                else super(TwitchVideosIE, cls).suitable(url))

    @staticmethod
    def _make_variables(channel_name, broadcast_type, sort):
        return {
            'channelOwnerLogin': channel_name,
            'broadcastType': broadcast_type,
            'videoSort': sort.upper(),
        }

    @staticmethod
    def _extract_entry(node):
        return _make_video_result(node)

    def _real_extract(self, url):
        channel_name = self._match_id(url)
        qs = parse_qs(url)
        filter = qs.get('filter', ['all'])[0]
        sort = qs.get('sort', ['time'])[0]
        broadcast = self._BROADCASTS.get(filter, self._DEFAULT_BROADCAST)
        return self.playlist_result(
            self._entries(channel_name, broadcast.type, sort),
            playlist_id=channel_name,
            playlist_title='%s - %s sorted by %s'
            % (channel_name, broadcast.label,
               self._SORTED_BY.get(sort, self._DEFAULT_SORTED_BY)))

class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
    _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:clips|videos/*?\?.*?\bfilter=clips)'

    _TESTS = [{
        # Clips
        'url': 'https://www.twitch.tv/vanillatv/clips?filter=clips&range=all',
        'info_dict': {
            'id': 'vanillatv',
            'title': 'vanillatv - Clips Top All',
        },
        'playlist_mincount': 1,
    }, {
        'url': 'https://www.twitch.tv/dota2ruhub/videos?filter=clips&range=7d',
        'only_matching': True,
    }]

    Clip = collections.namedtuple('Clip', ['filter', 'label'])

    _DEFAULT_CLIP = Clip('LAST_WEEK', 'Top 7D')
    _RANGE = {
        '24hr': Clip('LAST_DAY', 'Top 24H'),
        '7d': _DEFAULT_CLIP,
        '30d': Clip('LAST_MONTH', 'Top 30D'),
        'all': Clip('ALL_TIME', 'Top All'),
    }

    # NB: values other than 20 result in skipped videos
    _PAGE_LIMIT = 20

    _OPERATION_NAME = 'ClipsCards__User'
    _ENTRY_KIND = 'clip'
    _EDGE_KIND = 'ClipEdge'
    _NODE_KIND = 'Clip'

    @staticmethod
    def _make_variables(channel_name, filter):
        return {
            'login': channel_name,
            'criteria': {
                'filter': filter,
            },
        }

    @staticmethod
    def _extract_entry(node):
        assert isinstance(node, dict)
        clip_url = url_or_none(node.get('url'))
        if not clip_url:
            return
        return {
            '_type': 'url_transparent',
            'ie_key': TwitchClipsIE.ie_key(),
            'id': node.get('id'),
            'url': clip_url,
            'title': node.get('title'),
            'thumbnail': node.get('thumbnailURL'),
            'duration': float_or_none(node.get('durationSeconds')),
            'timestamp': unified_timestamp(node.get('createdAt')),
            'view_count': int_or_none(node.get('viewCount')),
            'language': node.get('language'),
        }

    def _real_extract(self, url):
        channel_name = self._match_id(url)
        qs = parse_qs(url)
        range = qs.get('range', ['7d'])[0]
        clip = self._RANGE.get(range, self._DEFAULT_CLIP)
        return self.playlist_result(
            self._entries(channel_name, clip.filter),
            playlist_id=channel_name,
            playlist_title='%s - Clips %s' % (channel_name, clip.label))

class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
    _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/videos/*?\?.*?\bfilter=collections'

    _TESTS = [{
        # Collections
        'url': 'https://www.twitch.tv/spamfish/videos?filter=collections',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - Collections',
        },
        'playlist_mincount': 3,
    }]

    _OPERATION_NAME = 'ChannelCollectionsContent'
    _ENTRY_KIND = 'collection'
    _EDGE_KIND = 'CollectionsItemEdge'
    _NODE_KIND = 'Collection'

    @staticmethod
    def _make_variables(channel_name):
        return {
            'ownerLogin': channel_name,
        }

    @staticmethod
    def _extract_entry(node):
        assert isinstance(node, dict)
        collection_id = node.get('id')
        if not collection_id:
            return
        return {
            '_type': 'url_transparent',
            'ie_key': TwitchCollectionIE.ie_key(),
            'id': collection_id,
            'url': 'https://www.twitch.tv/collections/%s' % collection_id,
            'title': node.get('title'),
            'thumbnail': node.get('thumbnailURL'),
            'duration': float_or_none(node.get('lengthSeconds')),
            'timestamp': unified_timestamp(node.get('updatedAt')),
            'view_count': int_or_none(node.get('viewCount')),
        }

    def _real_extract(self, url):
        channel_name = self._match_id(url)
        return self.playlist_result(
            self._entries(channel_name), playlist_id=channel_name,
            playlist_title='%s - Collections' % channel_name)

class TwitchStreamIE(TwitchBaseIE):
    IE_NAME = 'twitch:stream'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:(?:www|go|m)\.)?twitch\.tv/|
                            player\.twitch\.tv/\?.*?\bchannel=
                        )
                        (?P<id>[^/#?]+)
                    '''

    _TESTS = [{
        'url': 'http://www.twitch.tv/shroomztv',
        'info_dict': {
            'id': '12772022048',
            'display_id': 'shroomztv',
            'ext': 'mp4',
            'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
            'is_live': True,
            'timestamp': 1421928037,
            'upload_date': '20150122',
            'uploader': 'ShroomzTV',
            'uploader_id': 'shroomztv',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.twitch.tv/miracle_doto#profile-0',
        'only_matching': True,
    }, {
        'url': 'https://player.twitch.tv/?channel=lotsofs',
        'only_matching': True,
    }, {
        'url': 'https://go.twitch.tv/food',
        'only_matching': True,
    }, {
        'url': 'https://m.twitch.tv/food',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return (False
                if any(ie.suitable(url) for ie in (
                    TwitchVodIE,
                    TwitchCollectionIE,
                    TwitchVideosIE,
                    TwitchVideosClipsIE,
                    TwitchVideosCollectionsIE,
                    TwitchClipsIE))
                else super(TwitchStreamIE, cls).suitable(url))

    def _real_extract(self, url):
        channel_name = self._match_id(url).lower()

        gql = self._download_gql(
            channel_name, [{
                'operationName': 'StreamMetadata',
                'variables': {'channelLogin': channel_name},
            }, {
                'operationName': 'ComscoreStreamingQuery',
                'variables': {
                    'channel': channel_name,
                    'clipSlug': '',
                    'isClip': False,
                    'isLive': True,
                    'isVodOrCollection': False,
                    'vodID': '',
                },
            }, {
                'operationName': 'VideoPreviewOverlay',
                'variables': {'login': channel_name},
            }],
            'Downloading stream GraphQL')

        user = gql[0]['data']['user']

        if not user:
            raise ExtractorError(
                '%s does not exist' % channel_name, expected=True)

        stream = user['stream']

        if not stream:
            raise UserNotLive(video_id=channel_name)

        access_token = self._download_access_token(
            channel_name, 'stream', 'channelName')
        token = access_token['value']

        stream_id = stream.get('id') or channel_name
        query = {
            'allow_source': 'true',
            'allow_audio_only': 'true',
            'allow_spectre': 'true',
            'p': random.randint(1000000, 10000000),
            'player': 'twitchweb',
            'playlist_include_framerate': 'true',
            'segment_preference': '4',
            'sig': access_token['signature'].encode('utf-8'),
            'token': token.encode('utf-8'),
        }
        formats = self._extract_m3u8_formats(
            '%s/api/channel/hls/%s.m3u8' % (self._USHER_BASE, channel_name),
            stream_id, 'mp4', query=query)
        self._prefer_source(formats)

        view_count = stream.get('viewers')
        timestamp = unified_timestamp(stream.get('createdAt'))

        sq_user = try_get(gql, lambda x: x[1]['data']['user'], dict) or {}
        uploader = sq_user.get('displayName')
        description = try_get(
            sq_user, lambda x: x['broadcastSettings']['title'], compat_str)

        thumbnail = url_or_none(try_get(
            gql, lambda x: x[2]['data']['user']['stream']['previewImageURL'],
            compat_str))

        title = uploader or channel_name
        stream_type = stream.get('type')
        if stream_type in ['rerun', 'live']:
            title += ' (%s)' % stream_type

        return {
            'id': stream_id,
            'display_id': channel_name,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'uploader_id': channel_name,
            'timestamp': timestamp,
            'view_count': view_count,
            'formats': formats,
            'is_live': stream_type == 'live',
        }

class TwitchClipsIE(TwitchBaseIE):
    IE_NAME = 'twitch:clips'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            clips\.twitch\.tv/(?:embed\?.*?\bclip=|(?:[^/]+/)*)|
                            (?:(?:www|go|m)\.)?twitch\.tv/[^/]+/clip/
                        )
                        (?P<id>[^/?#&]+)
                    '''

    _TESTS = [{
        'url': 'https://clips.twitch.tv/FaintLightGullWholeWheat',
        'md5': '761769e1eafce0ffebfb4089cb3847cd',
        'info_dict': {
            'id': '42850523',
            'display_id': 'FaintLightGullWholeWheat',
            'ext': 'mp4',
            'title': 'EA Play 2016 Live from the Novo Theatre',
            'thumbnail': r're:^https?://.*\.jpg',
            'timestamp': 1465767393,
            'upload_date': '20160612',
            'creator': 'EA',
            'uploader': 'stereotype_',
            'uploader_id': '43566419',
        },
    }, {
        # multiple formats
        'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
        'only_matching': True,
    }, {
        'url': 'https://www.twitch.tv/sergeynixon/clip/StormyThankfulSproutFutureMan',
        'only_matching': True,
    }, {
        'url': 'https://clips.twitch.tv/embed?clip=InquisitiveBreakableYogurtJebaited',
        'only_matching': True,
    }, {
        'url': 'https://m.twitch.tv/rossbroadcast/clip/ConfidentBraveHumanChefFrank',
        'only_matching': True,
    }, {
        'url': 'https://go.twitch.tv/rossbroadcast/clip/ConfidentBraveHumanChefFrank',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        clip = self._download_gql(
            video_id, [{
                'operationName': 'VideoAccessToken_Clip',
                'variables': {
                    'slug': video_id,
                },
            }],
            'Downloading clip access token GraphQL')[0]['data']['clip']

        if not clip:
            raise ExtractorError(
                'This clip is no longer available', expected=True)

        access_query = {
            'sig': clip['playbackAccessToken']['signature'],
            'token': clip['playbackAccessToken']['value'],
        }

        data = self._download_base_gql(
            video_id, {
                'query': '''{
  clip(slug: "%s") {
    broadcaster {
      displayName
    }
    createdAt
    curator {
      displayName
      id
    }
    durationSeconds
    id
    tiny: thumbnailURL(width: 86, height: 45)
    small: thumbnailURL(width: 260, height: 147)
    medium: thumbnailURL(width: 480, height: 272)
    title
    videoQualities {
      frameRate
      quality
      sourceURL
    }
    viewCount
  }
}''' % video_id}, 'Downloading clip GraphQL', fatal=False)

        if data:
            clip = try_get(data, lambda x: x['data']['clip'], dict) or clip

        formats = []
        for option in clip.get('videoQualities', []):
            if not isinstance(option, dict):
                continue
            source = url_or_none(option.get('sourceURL'))
            if not source:
                continue
            formats.append({
                'url': update_url_query(source, access_query),
                'format_id': option.get('quality'),
                'height': int_or_none(option.get('quality')),
                'fps': int_or_none(option.get('frameRate')),
            })
        self._sort_formats(formats)

        thumbnails = []
        for thumbnail_id in ('tiny', 'small', 'medium'):
            thumbnail_url = clip.get(thumbnail_id)
            if not thumbnail_url:
                continue
            thumb = {
                'id': thumbnail_id,
                'url': thumbnail_url,
            }
            mobj = re.search(r'-(\d+)x(\d+)\.', thumbnail_url)
            if mobj:
                thumb.update({
                    'height': int(mobj.group(2)),
                    'width': int(mobj.group(1)),
                })
            thumbnails.append(thumb)

        old_id = self._search_regex(r'%7C(\d+)(?:-\d+)?.mp4', formats[-1]['url'], 'old id', default=None)

        return {
            'id': clip.get('id') or video_id,
            '_old_archive_ids': [make_archive_id(self, old_id)] if old_id else None,
            'display_id': video_id,
            'title': clip.get('title'),
            'formats': formats,
            'duration': int_or_none(clip.get('durationSeconds')),
            'view_count': int_or_none(clip.get('viewCount')),
            'timestamp': unified_timestamp(clip.get('createdAt')),
            'thumbnails': thumbnails,
            'creator': try_get(clip, lambda x: x['broadcaster']['displayName'], compat_str),
            'uploader': try_get(clip, lambda x: x['curator']['displayName'], compat_str),
            'uploader_id': try_get(clip, lambda x: x['curator']['id'], compat_str),
        }


code:
import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_parse_qs,
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    dict_get,
    ExtractorError,
    format_field,
    float_or_none,
    int_or_none,
    traverse_obj,
    try_get,
    strip_or_none,
    unified_timestamp,
    update_url_query,
    url_or_none,
    xpath_text,
)

from .periscope import (
    PeriscopeBaseIE,
    PeriscopeIE,
)

class TwitterBaseIE(InfoExtractor):
    _API_BASE = 'https://api.twitter.com/1.1/'
    _BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?twitter\.com/'
    _GUEST_TOKEN = None

    def _extract_variant_formats(self, variant, video_id):
        variant_url = variant.get('url')
        if not variant_url:
            return [], {}
        elif '.m3u8' in variant_url:
            return self._extract_m3u8_formats_and_subtitles(
                variant_url, video_id, 'mp4', 'm3u8_native',
                m3u8_id='hls', fatal=False)
        else:
            tbr = int_or_none(dict_get(variant, ('bitrate', 'bit_rate')), 1000) or None
            f = {
                'url': variant_url,
                'format_id': 'http' + ('-%d' % tbr if tbr else ''),
                'tbr': tbr,
            }
            self._search_dimensions_in_video_url(f, variant_url)
            return [f], {}

    def _extract_formats_from_vmap_url(self, vmap_url, video_id):
        vmap_url = url_or_none(vmap_url)
        if not vmap_url:
            return [], {}
        vmap_data = self._download_xml(vmap_url, video_id)
        formats = []
        subtitles = {}
        urls = []
        for video_variant in vmap_data.findall('.//{http://twitter.com/schema/videoVMapV2.xsd}videoVariant'):
            video_variant.attrib['url'] = compat_urllib_parse_unquote(
                video_variant.attrib['url'])
            urls.append(video_variant.attrib['url'])
            fmts, subs = self._extract_variant_formats(
                video_variant.attrib, video_id)
            formats.extend(fmts)
            subtitles = self._merge_subtitles(subtitles, subs)
        video_url = strip_or_none(xpath_text(vmap_data, './/MediaFile'))
        if video_url not in urls:
            fmts, subs = self._extract_variant_formats({'url': video_url}, video_id)
            formats.extend(fmts)
            subtitles = self._merge_subtitles(subtitles, subs)
        return formats, subtitles

    @staticmethod
    def _search_dimensions_in_video_url(a_format, video_url):
        m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
        if m:
            a_format.update({
                'width': int(m.group('width')),
                'height': int(m.group('height')),
            })

    def _call_api(self, path, video_id, query={}):
        headers = {
            'Authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw',
        }
        token = self._get_cookies(self._API_BASE).get('ct0')
        if token:
            headers['x-csrf-token'] = token.value
        if not self._GUEST_TOKEN:
            self._GUEST_TOKEN = self._download_json(
                self._API_BASE + 'guest/activate.json', video_id,
                'Downloading guest token', data=b'',
                headers=headers)['guest_token']
        headers['x-guest-token'] = self._GUEST_TOKEN
        try:
            return self._download_json(
                self._API_BASE + path, video_id, headers=headers, query=query)
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                raise ExtractorError(self._parse_json(
                    e.cause.read().decode(),
                    video_id)['errors'][0]['message'], expected=True)
            raise

class TwitterCardIE(InfoExtractor):
    IE_NAME = 'twitter:card'
    _VALID_URL = TwitterBaseIE._BASE_REGEX + r'i/(?:cards/tfw/v1|videos(?:/tweet)?)/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
            # MD5 checksums are different in different places
            'info_dict': {
                'id': '560070183650213889',
                'ext': 'mp4',
                'title': "Twitter - You can now shoot, edit and share video on Twitter. Capture life's most moving moments from your perspective.",
                'description': 'md5:18d3e24bb4f6e5007487dd546e53bd96',
                'uploader': 'Twitter',
                'uploader_id': 'Twitter',
                'thumbnail': r're:^https?://.*\.jpg',
                'duration': 30.033,
                'timestamp': 1422366112,
                'upload_date': '20150127',
            },
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
            'md5': '7137eca597f72b9abbe61e5ae0161399',
            'info_dict': {
                'id': '623160978427936768',
                'ext': 'mp4',
                'title': "NASA - Fly over Pluto's icy Norgay Mountains and Sputnik Plain in this @NASANewHorizons #PlutoFlyby video.",
                'description': "Fly over Pluto's icy Norgay Mountains and Sputnik Plain in this @NASANewHorizons #PlutoFlyby video. https://t.co/BJYgOjSeGA",
                'uploader': 'NASA',
                'uploader_id': 'NASA',
                'timestamp': 1437408129,
                'upload_date': '20150720',
            },
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
            'md5': 'b6d9683dd3f48e340ded81c0e917ad46',
            'info_dict': {
                'id': 'dq4Oj5quskI',
                'ext': 'mp4',
                'title': 'Ubuntu 11.10 Overview',
                'description': 'md5:a831e97fa384863d6e26ce48d1c43376',
                'upload_date': '20111013',
                'uploader': 'OMG! UBUNTU!',
                'uploader_id': 'omgubuntu',
            },
            'add_ie': ['Youtube'],
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
            'md5': '6dabeaca9e68cbb71c99c322a4b42a11',
            'info_dict': {
                'id': 'iBb2x00UVlv',
                'ext': 'mp4',
                'upload_date': '20151113',
                'uploader_id': '1189339351084113920',
                'uploader': 'ArsenalTerje',
                'title': 'Vine by ArsenalTerje',
                'timestamp': 1447451307,
            },
            'add_ie': ['Vine'],
        }, {
            'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
            'md5': '884812a2adc8aaf6fe52b15ccbfa3b88',
            'info_dict': {
                'id': '705235433198714880',
                'ext': 'mp4',
                'title': "Brent Yarina - Khalil Iverson's missed highlight dunk. And made highlight dunk. In one highlight.",
                'description': "Khalil Iverson's missed highlight dunk. And made highlight dunk. In one highlight. https://t.co/OrxcJ28Bns",
                'uploader': 'Brent Yarina',
                'uploader_id': 'BTNBrentYarina',
                'timestamp': 1456976204,
                'upload_date': '20160303',
            },
            'skip': 'This content is no longer available.',
        }, {
            'url': 'https://twitter.com/i/videos/752274308186120192',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        status_id = self._match_id(url)
        return self.url_result(
            'https://twitter.com/statuses/' + status_id,
            TwitterIE.ie_key(), status_id)

class TwitterIE(TwitterBaseIE):
    IE_NAME = 'twitter'
    _VALID_URL = TwitterBaseIE._BASE_REGEX + r'(?:(?:i/web|[^/]+)/status|statuses)/(?P<id>\d+)'

    _TESTS = [{
        'url': 'https://twitter.com/freethenipple/status/643211948184596480',
        'info_dict': {
            'id': '643211948184596480',
            'ext': 'mp4',
            'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': 'FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ',
            'uploader': 'FREE THE NIPPLE',
            'uploader_id': 'freethenipple',
            'duration': 12.922,
            'timestamp': 1442188653,
            'upload_date': '20150913',
            'age_limit': 18,
        },
    }, {
        'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
        'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',
        'info_dict': {
            'id': '657991469417025536',
            'ext': 'mp4',
            'title': 'Gifs - tu vai cai tu vai cai tu nao eh capaz disso tu vai cai',
            'description': 'Gifs on Twitter: "tu vai cai tu vai cai tu nao eh capaz disso tu vai cai https://t.co/tM46VHFlO5"',
            'thumbnail': r're:^https?://.*\.png',
            'uploader': 'Gifs',
            'uploader_id': 'giphz',
        },
        'expected_warnings': ['height', 'width'],
        'skip': 'Account suspended',
    }, {
        'url': 'https://twitter.com/starwars/status/665052190608723968',
        'info_dict': {
            'id': '665052190608723968',
            'ext': 'mp4',
            'title': 'Star Wars - A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens.',
            'description': 'A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens. https://t.co/OkSqT2fjWJ',
            'uploader_id': 'starwars',
            'uploader': 'Star Wars',
            'timestamp': 1447395772,
            'upload_date': '20151113',
        },
    }, {
        'url': 'https://twitter.com/BTNBrentYarina/status/705235433198714880',
        'info_dict': {
            'id': '705235433198714880',
            'ext': 'mp4',
            'title': "Brent Yarina - Khalil Iverson's missed highlight dunk. And made highlight dunk. In one highlight.",
            'description': "Khalil Iverson's missed highlight dunk. And made highlight dunk. In one highlight. https://t.co/OrxcJ28Bns",
            'uploader_id': 'BTNBrentYarina',
            'uploader': 'Brent Yarina',
            'timestamp': 1456976204,
            'upload_date': '20160303',
        },
        'params': {
            # The same video as https://twitter.com/i/videos/tweet/705235433198714880
            # Test case of TwitterCardIE
            'skip_download': True,
        },
    }, {
        'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
        'info_dict': {
            'id': '700207533655363584',
            'ext': 'mp4',
            'title': 'simon vertugo - BEAT PROD: @suhmeduh #Damndaniel',
            'description': 'BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ',
            'thumbnail': r're:^https?://.*\.jpg',
            'uploader': 'simon vertugo',
            'uploader_id': 'simonvertugo',
            'duration': 30.0,
            'timestamp': 1455777459,
            'upload_date': '20160218',
        },
    }, {
        'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
        'md5': '89a15ed345d13b86e9a5a5e051fa308a',
        'info_dict': {
            'id': 'MIOxnrUteUd',
            'ext': 'mp4',
            'title': 'Dr.Pepperの飲み方 #japanese #バカ #ドクペ #電動ガン',
            'uploader': 'TAKUMA',
            'uploader_id': '1004126642786242560',
            'timestamp': 1402826626,
            'upload_date': '20140615',
        },
        'add_ie': ['Vine'],
    }, {
        'url': 'https://twitter.com/captainamerica/status/719944021058060289',
        'info_dict': {
            'id': '719944021058060289',
            'ext': 'mp4',
            'title': 'Captain America - @King0fNerd Are you sure you made the right choice? Find out in theaters.',
            'description': '@King0fNerd Are you sure you made the right choice? Find out in theaters. https://t.co/GpgYi9xMJI',
            'uploader_id': 'CaptainAmerica',
            'uploader': 'Captain America',
            'duration': 3.17,
            'timestamp': 1460483005,
            'upload_date': '20160412',
        },
    }, {
        'url': 'https://twitter.com/OPP_HSD/status/779210622571536384',
        'info_dict': {
            'id': '1zqKVVlkqLaKB',
            'ext': 'mp4',
            'title': 'Sgt Kerry Schmidt - Ontario Provincial Police - Road rage, mischief, assault, rollover and fire in one occurrence',
            'upload_date': '20160923',
            'uploader_id': '1PmKqpJdOJQoY',
            'uploader': 'Sgt Kerry Schmidt - Ontario Provincial Police',
            'timestamp': 1474613214,
        },
        'add_ie': ['Periscope'],
    }, {
        # has mp4 formats via mobile API
        'url': 'https://twitter.com/news_al3alm/status/852138619213144067',
        'info_dict': {
            'id': '852138619213144067',
            'ext': 'mp4',
            'title': 'عالم الأخبار - كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة',
            'description': 'كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة   https://t.co/xg6OhpyKfN',
            'uploader': 'عالم الأخبار',
            'uploader_id': 'news_al3alm',
            'duration': 277.4,
            'timestamp': 1492000653,
            'upload_date': '20170412',
        },
        'skip': 'Account suspended',
    }, {
        'url': 'https://twitter.com/i/web/status/910031516746514432',
        'info_dict': {
            'id': '910031516746514432',
            'ext': 'mp4',
            'title': 'Préfet de Guadeloupe - [Direct] #Maria Le centre se trouve actuellement au sud de Basse-Terre. Restez confinés. Réfugiez-vous dans la pièce la + sûre.',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': '[Direct] #Maria Le centre se trouve actuellement au sud de Basse-Terre. Restez confinés. Réfugiez-vous dans la pièce la + sûre. https://t.co/mwx01Rs4lo',
            'uploader': 'Préfet de Guadeloupe',
            'uploader_id': 'Prefet971',
            'duration': 47.48,
            'timestamp': 1505803395,
            'upload_date': '20170919',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        },
    }, {
        # card via api.twitter.com/1.1/videos/tweet/config
        'url': 'https://twitter.com/LisPower1/status/1001551623938805763',
        'info_dict': {
            'id': '1001551623938805763',
            'ext': 'mp4',
            'title': 're:.*?Shep is on a roll today.*?',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': 'md5:37b9f2ff31720cef23b2bd42ee8a0f09',
            'uploader': 'Lis Power',
            'uploader_id': 'LisPower1',
            'duration': 111.278,
            'timestamp': 1527623489,
            'upload_date': '20180529',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        },
    }, {
        'url': 'https://twitter.com/foobar/status/1087791357756956680',
        'info_dict': {
            'id': '1087791357756956680',
            'ext': 'mp4',
            'title': 'Twitter - A new is coming.  Some of you got an opt-in to try it now. Check out the emoji button, quick keyboard shortcuts, upgraded trends, advanced search, and more. Let us know your thoughts!',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': 'md5:6dfd341a3310fb97d80d2bf7145df976',
            'uploader': 'Twitter',
            'uploader_id': 'Twitter',
            'duration': 61.567,
            'timestamp': 1548184644,
            'upload_date': '20190122',
        },
    }, {
        # not available in Periscope
        'url': 'https://twitter.com/ViviEducation/status/1136534865145286656',
        'info_dict': {
            'id': '1vOGwqejwoWxB',
            'ext': 'mp4',
            'title': 'Vivi - Vivi founder @lior_rauchy announcing our new student feedback tool live at @EduTECH_AU #EduTECH2019',
            'uploader': 'Vivi',
            'uploader_id': '1eVjYOLGkGrQL',
        },
        'add_ie': ['TwitterBroadcast'],
    }, {
        # unified card
        'url': 'https://twitter.com/BrooklynNets/status/1349794411333394432?s=20',
        'info_dict': {
            'id': '1349794411333394432',
            'ext': 'mp4',
            'title': 'md5:d1c4941658e4caaa6cb579260d85dcba',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': 'md5:71ead15ec44cee55071547d6447c6a3e',
            'uploader': 'Brooklyn Nets',
            'uploader_id': 'BrooklynNets',
            'duration': 324.484,
            'timestamp': 1610651040,
            'upload_date': '20210114',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # Twitch Clip Embed
        'url': 'https://twitter.com/GunB1g/status/1163218564784017422',
        'only_matching': True,
    }, {
        # promo_video_website card
        'url': 'https://twitter.com/GunB1g/status/1163218564784017422',
        'only_matching': True,
    }, {
        # promo_video_convo card
        'url': 'https://twitter.com/poco_dandy/status/1047395834013384704',
        'only_matching': True,
    }, {
        # appplayer card
        'url': 'https://twitter.com/poco_dandy/status/1150646424461176832',
        'only_matching': True,
    }, {
        # video_direct_message card
        'url': 'https://twitter.com/qarev001/status/1348948114569269251',
        'only_matching': True,
    }, {
        # poll2choice_video card
        'url': 'https://twitter.com/CAF_Online/status/1349365911120195585',
        'only_matching': True,
    }, {
        # poll3choice_video card
        'url': 'https://twitter.com/SamsungMobileSA/status/1348609186725289984',
        'only_matching': True,
    }, {
        # poll4choice_video card
        'url': 'https://twitter.com/SouthamptonFC/status/1347577658079641604',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        twid = self._match_id(url)
        status = self._call_api(
            'statuses/show/%s.json' % twid, twid, {
                'cards_platform': 'Web-12',
                'include_cards': 1,
                'include_reply_count': 1,
                'include_user_entities': 0,
                'tweet_mode': 'extended',
            })

        title = description = status['full_text'].replace('\n', ' ')
        # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
        title = re.sub(r'\s+(https?://[^ ]+)', '', title)
        user = status.get('user') or {}
        uploader = user.get('name')
        if uploader:
            title = '%s - %s' % (uploader, title)
        uploader_id = user.get('screen_name')

        tags = []
        for hashtag in (try_get(status, lambda x: x['entities']['hashtags'], list) or []):
            hashtag_text = hashtag.get('text')
            if not hashtag_text:
                continue
            tags.append(hashtag_text)

        info = {
            'id': twid,
            'title': title,
            'description': description,
            'uploader': uploader,
            'timestamp': unified_timestamp(status.get('created_at')),
            'uploader_id': uploader_id,
            'uploader_url': format_field(uploader_id, None, 'https://twitter.com/%s'),
            'like_count': int_or_none(status.get('favorite_count')),
            'repost_count': int_or_none(status.get('retweet_count')),
            'comment_count': int_or_none(status.get('reply_count')),
            'age_limit': 18 if status.get('possibly_sensitive') else 0,
            'tags': tags,
        }

        def extract_from_video_info(media):
            video_info = media.get('video_info') or {}

            formats = []
            subtitles = {}
            for variant in video_info.get('variants', []):
                fmts, subs = self._extract_variant_formats(variant, twid)
                subtitles = self._merge_subtitles(subtitles, subs)
                formats.extend(fmts)
            self._sort_formats(formats, ('res', 'br', 'size', 'proto'))  # The codec of http formats are unknown

            thumbnails = []
            media_url = media.get('media_url_https') or media.get('media_url')
            if media_url:
                def add_thumbnail(name, size):
                    thumbnails.append({
                        'id': name,
                        'url': update_url_query(media_url, {'name': name}),
                        'width': int_or_none(size.get('w') or size.get('width')),
                        'height': int_or_none(size.get('h') or size.get('height')),
                    })
                for name, size in media.get('sizes', {}).items():
                    add_thumbnail(name, size)
                add_thumbnail('orig', media.get('original_info') or {})

            info.update({
                'formats': formats,
                'subtitles': subtitles,
                'thumbnails': thumbnails,
                'duration': float_or_none(video_info.get('duration_millis'), 1000),
            })

        media = traverse_obj(status, ((None, 'quoted_status'), 'extended_entities', 'media', 0), get_all=False)
        if media and media.get('type') != 'photo':
            extract_from_video_info(media)
        else:
            card = status.get('card')
            if card:
                binding_values = card['binding_values']

                def get_binding_value(k):
                    o = binding_values.get(k) or {}
                    return try_get(o, lambda x: x[x['type'].lower() + '_value'])

                card_name = card['name'].split(':')[-1]
                if card_name == 'player':
                    info.update({
                        '_type': 'url',
                        'url': get_binding_value('player_url'),
                    })
                elif card_name == 'periscope_broadcast':
                    info.update({
                        '_type': 'url',
                        'url': get_binding_value('url') or get_binding_value('player_url'),
                        'ie_key': PeriscopeIE.ie_key(),
                    })
                elif card_name == 'broadcast':
                    info.update({
                        '_type': 'url',
                        'url': get_binding_value('broadcast_url'),
                        'ie_key': TwitterBroadcastIE.ie_key(),
                    })
                elif card_name == 'summary':
                    info.update({
                        '_type': 'url',
                        'url': get_binding_value('card_url'),
                    })
                elif card_name == 'unified_card':
                    media_entities = self._parse_json(get_binding_value('unified_card'), twid)['media_entities']
                    extract_from_video_info(next(iter(media_entities.values())))
                # amplify, promo_video_website, promo_video_convo, appplayer,
                # video_direct_message, poll2choice_video, poll3choice_video,
                # poll4choice_video, ...
                else:
                    is_amplify = card_name == 'amplify'
                    vmap_url = get_binding_value('amplify_url_vmap') if is_amplify else get_binding_value('player_stream_url')
                    content_id = get_binding_value('%s_content_id' % (card_name if is_amplify else 'player'))
                    formats, subtitles = self._extract_formats_from_vmap_url(vmap_url, content_id or twid)
                    self._sort_formats(formats)

                    thumbnails = []
                    for suffix in ('_small', '', '_large', '_x_large', '_original'):
                        image = get_binding_value('player_image' + suffix) or {}
                        image_url = image.get('url')
                        if not image_url or '/player-placeholder' in image_url:
                            continue
                        thumbnails.append({
                            'id': suffix[1:] if suffix else 'medium',
                            'url': image_url,
                            'width': int_or_none(image.get('width')),
                            'height': int_or_none(image.get('height')),
                        })

                    info.update({
                        'formats': formats,
                        'subtitles': subtitles,
                        'thumbnails': thumbnails,
                        'duration': int_or_none(get_binding_value(
                            'content_duration_seconds')),
                    })
            else:
                expanded_url = try_get(status, lambda x: x['entities']['urls'][0]['expanded_url'])
                if not expanded_url:
                    raise ExtractorError("There's no video in this tweet.")
                info.update({
                    '_type': 'url',
                    'url': expanded_url,
                })
        return info

class TwitterAmplifyIE(TwitterBaseIE):
    IE_NAME = 'twitter:amplify'
    _VALID_URL = r'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'

    _TEST = {
        'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
        'md5': '7df102d0b9fd7066b86f3159f8e81bf6',
        'info_dict': {
            'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
            'ext': 'mp4',
            'title': 'Twitter Video',
            'thumbnail': 're:^https?://.*',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        vmap_url = self._html_search_meta(
            'twitter:amplify:vmap', webpage, 'vmap url')
        formats = self._extract_formats_from_vmap_url(vmap_url, video_id)

        thumbnails = []
        thumbnail = self._html_search_meta(
            'twitter:image:src', webpage, 'thumbnail', fatal=False)

        def _find_dimension(target):
            w = int_or_none(self._html_search_meta(
                'twitter:%s:width' % target, webpage, fatal=False))
            h = int_or_none(self._html_search_meta(
                'twitter:%s:height' % target, webpage, fatal=False))
            return w, h

        if thumbnail:
            thumbnail_w, thumbnail_h = _find_dimension('image')
            thumbnails.append({
                'url': thumbnail,
                'width': thumbnail_w,
                'height': thumbnail_h,
            })

        video_w, video_h = _find_dimension('player')
        formats[0].update({
            'width': video_w,
            'height': video_h,
        })

        return {
            'id': video_id,
            'title': 'Twitter Video',
            'formats': formats,
            'thumbnails': thumbnails,
        }

class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE):
    IE_NAME = 'twitter:broadcast'
    _VALID_URL = TwitterBaseIE._BASE_REGEX + r'i/broadcasts/(?P<id>[0-9a-zA-Z]{13})'

    _TEST = {
        # untitled Periscope video
        'url': 'https://twitter.com/i/broadcasts/1yNGaQLWpejGj',
        'info_dict': {
            'id': '1yNGaQLWpejGj',
            'ext': 'mp4',
            'title': 'Andrea May Sahouri - Periscope Broadcast',
            'uploader': 'Andrea May Sahouri',
            'uploader_id': '1PXEdBZWpGwKe',
        },
    }

    def _real_extract(self, url):
        broadcast_id = self._match_id(url)
        broadcast = self._call_api(
            'broadcasts/show.json', broadcast_id,
            {'ids': broadcast_id})['broadcasts'][broadcast_id]
        info = self._parse_broadcast_data(broadcast, broadcast_id)
        media_key = broadcast['media_key']
        source = self._call_api(
            'live_video_stream/status/' + media_key, media_key)['source']
        m3u8_url = source.get('noRedirectPlaybackUrl') or source['location']
        if '/live_video_stream/geoblocked/' in m3u8_url:
            self.raise_geo_restricted()
        m3u8_id = compat_parse_qs(compat_urllib_parse_urlparse(
            m3u8_url).query).get('type', [None])[0]
        state, width, height = self._extract_common_format_info(broadcast)
        info['formats'] = self._extract_pscp_m3u8_formats(
            m3u8_url, broadcast_id, m3u8_id, state, width, height)
        return info

class TwitterShortenerIE(TwitterBaseIE):
    IE_NAME = 'twitter:shortener'
    _VALID_URL = r'https?://t.co/(?P<id>[^?]+)|tco:(?P<eid>[^?]+)'
    _BASE_URL = 'https://t.co/'

    def _real_extract(self, url):
        mobj = self._match_valid_url(url)
        eid, id = mobj.group('eid', 'id')
        if eid:
            id = eid
            url = self._BASE_URL + id
        new_url = self._request_webpage(url, id, headers={'User-Agent': 'curl'}).geturl()
        __UNSAFE_LINK = "https://twitter.com/safety/unsafe_link_warning?unsafe_link="
        if new_url.startswith(__UNSAFE_LINK):
            new_url = new_url.replace(__UNSAFE_LINK, "")
        return self.url_result(new_url)

repo_name: Lesmiscore/ytdl


code:
import re
import urllib.request

from .common import InfoExtractor
from ..compat import compat_HTTPError, compat_str, compat_urlparse
from ..utils import (
    ExtractorError,
    determine_ext,
    extract_attributes,
    float_or_none,
    int_or_none,
    js_to_json,
    sanitized_Request,
    try_get,
    unescapeHTML,
    url_or_none,
    urlencode_postdata,
)

class UdemyIE(InfoExtractor):
    IE_NAME = 'udemy'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:[^/]+\.)?udemy\.com/
                        (?:
                            [^#]+\#/lecture/|
                            lecture/view/?\?lectureId=|
                            [^/]+/learn/v4/t/lecture/
                        )
                        (?P<id>\d+)
                    '''
    _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
    _ORIGIN_URL = 'https://www.udemy.com'
    _NETRC_MACHINE = 'udemy'

    _TESTS = [{
        'url': 'https://www.udemy.com/java-tutorial/#/lecture/172757',
        'md5': '98eda5b657e752cf945d8445e261b5c5',
        'info_dict': {
            'id': '160614',
            'ext': 'mp4',
            'title': 'Introduction and Installation',
            'description': 'md5:c0d51f6f21ef4ec65f091055a5eef876',
            'duration': 579.29,
        },
        'skip': 'Requires udemy account credentials',
    }, {
        # new URL schema
        'url': 'https://www.udemy.com/electric-bass-right-from-the-start/learn/v4/t/lecture/4580906',
        'only_matching': True,
    }, {
        # no url in outputs format entry
        'url': 'https://www.udemy.com/learn-web-development-complete-step-by-step-guide-to-success/learn/v4/t/lecture/4125812',
        'only_matching': True,
    }, {
        # only outputs rendition
        'url': 'https://www.udemy.com/how-you-can-help-your-local-community-5-amazing-examples/learn/v4/t/lecture/3225750?start=0',
        'only_matching': True,
    }, {
        'url': 'https://wipro.udemy.com/java-tutorial/#/lecture/172757',
        'only_matching': True,
    }]

    def _extract_course_info(self, webpage, video_id):
        course = self._parse_json(
            unescapeHTML(self._search_regex(
                r'ng-init=["\'].*\bcourse=({.+?})[;"\']',
                webpage, 'course', default='{}')),
            video_id, fatal=False) or {}
        course_id = course.get('id') or self._search_regex(
            [
                r'data-course-id=["\'](\d+)',
                r'"courseId"\s*:\s*(\d+)'
            ], webpage, 'course id')
        return course_id, course.get('title')

    def _enroll_course(self, base_url, webpage, course_id):
        def combine_url(base_url, url):
            return compat_urlparse.urljoin(base_url, url) if not url.startswith('http') else url

        checkout_url = unescapeHTML(self._search_regex(
            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/(?:payment|cart)/checkout/.+?)\1',
            webpage, 'checkout url', group='url', default=None))
        if checkout_url:
            raise ExtractorError(
                'Course %s is not free. You have to pay for it before you can download. '
                'Use this URL to confirm purchase: %s'
                % (course_id, combine_url(base_url, checkout_url)),
                expected=True)

        enroll_url = unescapeHTML(self._search_regex(
            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/course/subscribe/.+?)\1',
            webpage, 'enroll url', group='url', default=None))
        if enroll_url:
            webpage = self._download_webpage(
                combine_url(base_url, enroll_url),
                course_id, 'Enrolling in the course',
                headers={'Referer': base_url})
            if '>You have enrolled in' in webpage:
                self.to_screen('%s: Successfully enrolled in the course' % course_id)

    def _download_lecture(self, course_id, lecture_id):
        return self._download_json(
            'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?'
            % (course_id, lecture_id),
            lecture_id, 'Downloading lecture JSON', query={
                'fields[lecture]': 'title,description,view_html,asset',
                'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,stream_urls,captions,data',
            })

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            error_str = 'Udemy returned error #%s: %s' % (error.get('code'), error.get('message'))
            error_data = error.get('data')
            if error_data:
                error_str += ' - %s' % error_data.get('formErrors')
            raise ExtractorError(error_str, expected=True)

    def _download_webpage_handle(self, *args, **kwargs):
        headers = kwargs.get('headers', {}).copy()
        headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'
        kwargs['headers'] = headers
        ret = super(UdemyIE, self)._download_webpage_handle(
            *args, **kwargs)
        if not ret:
            return ret
        webpage, _ = ret
        if any(p in webpage for p in (
                '>Please verify you are a human',
                'Access to this page has been denied because we believe you are using automation tools to browse the website',
                '"_pxCaptcha"')):
            raise ExtractorError(
                'Udemy asks you to solve a CAPTCHA. Login with browser, '
                'solve CAPTCHA, then export cookies and pass cookie file to '
                'yt-dlp with --cookies.', expected=True)
        return ret

    def _download_json(self, url_or_request, *args, **kwargs):
        headers = {
            'X-Udemy-Snail-Case': 'true',
            'X-Requested-With': 'XMLHttpRequest',
        }
        for cookie in self.cookiejar:
            if cookie.name == 'client_id':
                headers['X-Udemy-Client-Id'] = cookie.value
            elif cookie.name == 'access_token':
                headers['X-Udemy-Bearer-Token'] = cookie.value
                headers['X-Udemy-Authorization'] = 'Bearer %s' % cookie.value

        if isinstance(url_or_request, urllib.request.Request):
            for header, value in headers.items():
                url_or_request.add_header(header, value)
        else:
            url_or_request = sanitized_Request(url_or_request, headers=headers)

        response = super(UdemyIE, self)._download_json(url_or_request, *args, **kwargs)
        self._handle_error(response)
        return response

    def _perform_login(self, username, password):
        login_popup = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login popup')

        def is_logged(webpage):
            return any(re.search(p, webpage) for p in (
                r'href=["\'](?:https://www\.udemy\.com)?/user/logout/',
                r'>Logout<'))

        # already logged in
        if is_logged(login_popup):
            return

        login_form = self._form_hidden_inputs('login-form', login_popup)

        login_form.update({
            'email': username,
            'password': password,
        })

        response = self._download_webpage(
            self._LOGIN_URL, None, 'Logging in',
            data=urlencode_postdata(login_form),
            headers={
                'Referer': self._ORIGIN_URL,
                'Origin': self._ORIGIN_URL,
            })

        if not is_logged(response):
            error = self._html_search_regex(
                r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
                response, 'error message', default=None)
            if error:
                raise ExtractorError('Unable to login: %s' % error, expected=True)
            raise ExtractorError('Unable to log in')

    def _real_extract(self, url):
        lecture_id = self._match_id(url)

        webpage = self._download_webpage(url, lecture_id)

        course_id, _ = self._extract_course_info(webpage, lecture_id)

        try:
            lecture = self._download_lecture(course_id, lecture_id)
        except ExtractorError as e:
            # Error could possibly mean we are not enrolled in the course
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                self._enroll_course(url, webpage, course_id)
                lecture = self._download_lecture(course_id, lecture_id)
            else:
                raise

        title = lecture['title']
        description = lecture.get('description')

        asset = lecture['asset']

        asset_type = asset.get('asset_type') or asset.get('assetType')
        if asset_type != 'Video':
            raise ExtractorError(
                'Lecture %s is not a video' % lecture_id, expected=True)

        stream_url = asset.get('stream_url') or asset.get('streamUrl')
        if stream_url:
            youtube_url = self._search_regex(
                r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None)
            if youtube_url:
                return self.url_result(youtube_url, 'Youtube')

        video_id = compat_str(asset['id'])
        thumbnail = asset.get('thumbnail_url') or asset.get('thumbnailUrl')
        duration = float_or_none(asset.get('data', {}).get('duration'))

        subtitles = {}
        automatic_captions = {}

        formats = []

        def extract_output_format(src, f_id):
            return {
                'url': src.get('url'),
                'format_id': '%sp' % (src.get('height') or f_id),
                'width': int_or_none(src.get('width')),
                'height': int_or_none(src.get('height')),
                'vbr': int_or_none(src.get('video_bitrate_in_kbps')),
                'vcodec': src.get('video_codec'),
                'fps': int_or_none(src.get('frame_rate')),
                'abr': int_or_none(src.get('audio_bitrate_in_kbps')),
                'acodec': src.get('audio_codec'),
                'asr': int_or_none(src.get('audio_sample_rate')),
                'tbr': int_or_none(src.get('total_bitrate_in_kbps')),
                'filesize': int_or_none(src.get('file_size_in_bytes')),
            }

        outputs = asset.get('data', {}).get('outputs')
        if not isinstance(outputs, dict):
            outputs = {}

        def add_output_format_meta(f, key):
            output = outputs.get(key)
            if isinstance(output, dict):
                output_format = extract_output_format(output, key)
                output_format.update(f)
                return output_format
            return f

        def extract_formats(source_list):
            if not isinstance(source_list, list):
                return
            for source in source_list:
                video_url = url_or_none(source.get('file') or source.get('src'))
                if not video_url:
                    continue
                if source.get('type') == 'application/x-mpegURL' or determine_ext(video_url) == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id='hls', fatal=False))
                    continue
                format_id = source.get('label')
                f = {
                    'url': video_url,
                    'format_id': '%sp' % format_id,
                    'height': int_or_none(format_id),
                }
                if format_id:
                    # Some videos contain additional metadata (e.g.
                    # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
                    f = add_output_format_meta(f, format_id)
                formats.append(f)

        def extract_subtitles(track_list):
            if not isinstance(track_list, list):
                return
            for track in track_list:
                if not isinstance(track, dict):
                    continue
                if track.get('kind') != 'captions':
                    continue
                src = url_or_none(track.get('src'))
                if not src:
                    continue
                lang = track.get('language') or track.get(
                    'srclang') or track.get('label')
                sub_dict = automatic_captions if track.get(
                    'autogenerated') is True else subtitles
                sub_dict.setdefault(lang, []).append({
                    'url': src,
                })

        for url_kind in ('download', 'stream'):
            urls = asset.get('%s_urls' % url_kind)
            if isinstance(urls, dict):
                extract_formats(urls.get('Video'))

        captions = asset.get('captions')
        if isinstance(captions, list):
            for cc in captions:
                if not isinstance(cc, dict):
                    continue
                cc_url = url_or_none(cc.get('url'))
                if not cc_url:
                    continue
                lang = try_get(cc, lambda x: x['locale']['locale'], compat_str)
                sub_dict = (automatic_captions if cc.get('source') == 'auto'
                            else subtitles)
                sub_dict.setdefault(lang or 'en', []).append({
                    'url': cc_url,
                })

        view_html = lecture.get('view_html')
        if view_html:
            view_html_urls = set()
            for source in re.findall(r'<source[^>]+>', view_html):
                attributes = extract_attributes(source)
                src = attributes.get('src')
                if not src:
                    continue
                res = attributes.get('data-res')
                height = int_or_none(res)
                if src in view_html_urls:
                    continue
                view_html_urls.add(src)
                if attributes.get('type') == 'application/x-mpegURL' or determine_ext(src) == 'm3u8':
                    m3u8_formats = self._extract_m3u8_formats(
                        src, video_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id='hls', fatal=False)
                    for f in m3u8_formats:
                        m = re.search(r'/hls_(?P<height>\d{3,4})_(?P<tbr>\d{2,})/', f['url'])
                        if m:
                            if not f.get('height'):
                                f['height'] = int(m.group('height'))
                            if not f.get('tbr'):
                                f['tbr'] = int(m.group('tbr'))
                    formats.extend(m3u8_formats)
                else:
                    formats.append(add_output_format_meta({
                        'url': src,
                        'format_id': '%dp' % height if height else None,
                        'height': height,
                    }, res))

            # react rendition since 2017.04.15 (see
            # https://github.com/ytdl-org/youtube-dl/issues/12744)
            data = self._parse_json(
                self._search_regex(
                    r'videojs-setup-data=(["\'])(?P<data>{.+?})\1', view_html,
                    'setup data', default='{}', group='data'), video_id,
                transform_source=unescapeHTML, fatal=False)
            if data and isinstance(data, dict):
                extract_formats(data.get('sources'))
                if not duration:
                    duration = int_or_none(data.get('duration'))
                extract_subtitles(data.get('tracks'))

            if not subtitles and not automatic_captions:
                text_tracks = self._parse_json(
                    self._search_regex(
                        r'text-tracks=(["\'])(?P<data>\[.+?\])\1', view_html,
                        'text tracks', default='{}', group='data'), video_id,
                    transform_source=lambda s: js_to_json(unescapeHTML(s)),
                    fatal=False)
                extract_subtitles(text_tracks)

        if not formats and outputs:
            for format_id, output in outputs.items():
                f = extract_output_format(output, format_id)
                if f.get('url'):
                    formats.append(f)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
            'subtitles': subtitles,
            'automatic_captions': automatic_captions,
        }

class UdemyCourseIE(UdemyIE):
    IE_NAME = 'udemy:course'
    _VALID_URL = r'https?://(?:[^/]+\.)?udemy\.com/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://www.udemy.com/java-tutorial/',
        'only_matching': True,
    }, {
        'url': 'https://wipro.udemy.com/java-tutorial/',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if UdemyIE.suitable(url) else super(UdemyCourseIE, cls).suitable(url)

    def _real_extract(self, url):
        course_path = self._match_id(url)

        webpage = self._download_webpage(url, course_path)

        course_id, title = self._extract_course_info(webpage, course_path)

        self._enroll_course(url, webpage, course_id)

        response = self._download_json(
            'https://www.udemy.com/api-2.0/courses/%s/cached-subscriber-curriculum-items' % course_id,
            course_id, 'Downloading course curriculum', query={
                'fields[chapter]': 'title,object_index',
                'fields[lecture]': 'title,asset',
                'page_size': '1000',
            })

        entries = []
        chapter, chapter_number = [None] * 2
        for entry in response['results']:
            clazz = entry.get('_class')
            if clazz == 'lecture':
                asset = entry.get('asset')
                if isinstance(asset, dict):
                    asset_type = asset.get('asset_type') or asset.get('assetType')
                    if asset_type != 'Video':
                        continue
                lecture_id = entry.get('id')
                if lecture_id:
                    entry = {
                        '_type': 'url_transparent',
                        'url': 'https://www.udemy.com/%s/learn/v4/t/lecture/%s' % (course_path, entry['id']),
                        'title': entry.get('title'),
                        'ie_key': UdemyIE.ie_key(),
                    }
                    if chapter_number:
                        entry['chapter_number'] = chapter_number
                    if chapter:
                        entry['chapter'] = chapter
                    entries.append(entry)
            elif clazz == 'chapter':
                chapter_number = entry.get('object_index')
                chapter = entry.get('title')

        return self.playlist_result(entries, course_id, title)


code:
import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    int_or_none,
    js_to_json,
)
from ..compat import compat_urlparse

class UDNEmbedIE(InfoExtractor):
    IE_DESC = '聯合影音'
    _PROTOCOL_RELATIVE_VALID_URL = r'//video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
    _VALID_URL = r'https?:' + _PROTOCOL_RELATIVE_VALID_URL
    _EMBED_REGEX = [r'<iframe[^>]+src="(?:https?:)?(?P<url>%s)"' % _PROTOCOL_RELATIVE_VALID_URL]
    _TESTS = [{
        'url': 'http://video.udn.com/embed/news/300040',
        'info_dict': {
            'id': '300040',
            'ext': 'mp4',
            'title': '生物老師男變女 全校挺"做自己"',
            'thumbnail': r're:^https?://.*\.jpg$',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'expected_warnings': ['Failed to parse JSON Expecting value'],
    }, {
        'url': 'https://video.udn.com/embed/news/300040',
        'only_matching': True,
    }, {
        # From https://video.udn.com/news/303776
        'url': 'https://video.udn.com/play/news/303776',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        page = self._download_webpage(url, video_id)

        options_str = self._html_search_regex(
            r'var\s+options\s*=\s*([^;]+);', page, 'options')
        trans_options_str = js_to_json(options_str)
        options = self._parse_json(trans_options_str, 'options', fatal=False) or {}
        if options:
            video_urls = options['video']
            title = options['title']
            poster = options.get('poster')
        else:
            video_urls = self._parse_json(self._html_search_regex(
                r'"video"\s*:\s*({.+?})\s*,', trans_options_str, 'video urls'), 'video urls')
            title = self._html_search_regex(
                r"title\s*:\s*'(.+?)'\s*,", options_str, 'title')
            poster = self._html_search_regex(
                r"poster\s*:\s*'(.+?)'\s*,", options_str, 'poster', default=None)

        if video_urls.get('youtube'):
            return self.url_result(video_urls.get('youtube'), 'Youtube')

        formats = []
        for video_type, api_url in video_urls.items():
            if not api_url:
                continue

            video_url = self._download_webpage(
                compat_urlparse.urljoin(url, api_url), video_id,
                note='retrieve url for %s video' % video_type)

            ext = determine_ext(video_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, ext='mp4', m3u8_id='hls'))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    video_url, video_id, f4m_id='hds'))
            else:
                mobj = re.search(r'_(?P<height>\d+)p_(?P<tbr>\d+)\.mp4', video_url)
                a_format = {
                    'url': video_url,
                    # video_type may be 'mp4', which confuses YoutubeDL
                    'format_id': 'http-' + video_type,
                }
                if mobj:
                    a_format.update({
                        'height': int_or_none(mobj.group('height')),
                        'tbr': int_or_none(mobj.group('tbr')),
                    })
                formats.append(a_format)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'thumbnail': poster,
        }


code:
from .imggaming import ImgGamingBaseIE

class UFCTVIE(ImgGamingBaseIE):
    _VALID_URL = ImgGamingBaseIE._VALID_URL_TEMPL % r'(?:(?:app|www)\.)?(?:ufc\.tv|(?:ufc)?fightpass\.com)|ufcfightpass\.img(?:dge|gaming)\.com'
    _NETRC_MACHINE = 'ufctv'
    _REALM = 'ufc'

class UFCArabiaIE(ImgGamingBaseIE):
    _VALID_URL = ImgGamingBaseIE._VALID_URL_TEMPL % r'(?:(?:app|www)\.)?ufcarabia\.(?:ae|com)'
    _NETRC_MACHINE = 'ufcarabia'
    _REALM = 'admufc'


code:
from ..utils import (
    unescapeHTML,
    urljoin,
    ExtractorError,
)
from .common import InfoExtractor
from .vimeo import VimeoIE
from .youtube import YoutubeIE

class UkColumnIE(InfoExtractor):
    IE_NAME = 'ukcolumn'
    _VALID_URL = r'(?i)https?://(?:www\.)?ukcolumn\.org(/index\.php)?/(?:video|ukcolumn-news)/(?P<id>[-a-z0-9]+)'

    _TESTS = [{
        'url': 'https://www.ukcolumn.org/ukcolumn-news/uk-column-news-28th-april-2021',
        'info_dict': {
            'id': '541632443',
            'ext': 'mp4',
            'title': 'UK Column News - 28th April 2021',
            'uploader_id': 'ukcolumn',
            'uploader': 'UK Column',
        },
        'add_ie': [VimeoIE.ie_key()],
        'expected_warnings': ['Unable to download JSON metadata'],
        'params': {
            'skip_download': 'Handled by Vimeo',
        },
    }, {
        'url': 'https://www.ukcolumn.org/video/insight-eu-military-unification',
        'info_dict': {
            'id': 'Fzbnb9t7XAw',
            'ext': 'mp4',
            'title': 'Insight: EU Military Unification',
            'uploader_id': 'ukcolumn',
            'description': 'md5:29a207965271af89baa0bc191f5de576',
            'uploader': 'UK Column',
            'upload_date': '20170514',
        },
        'add_ie': [YoutubeIE.ie_key()],
        'params': {
            'skip_download': 'Handled by Youtube',
        },
    }, {
        'url': 'https://www.ukcolumn.org/index.php/ukcolumn-news/uk-column-news-30th-april-2021',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        oembed_url = urljoin(url, unescapeHTML(self._search_regex(
            r'<iframe[^>]+src=(["\'])(?P<url>/media/oembed\?url=.+?)\1',
            webpage, 'OEmbed URL', group='url')))
        oembed_webpage = self._download_webpage(
            oembed_url, display_id, note='Downloading OEmbed page')

        ie, video_url = YoutubeIE, YoutubeIE._extract_url(oembed_webpage)
        if not video_url:
            ie, video_url = VimeoIE, VimeoIE._extract_url(url, oembed_webpage)
        if not video_url:
            raise ExtractorError('No embedded video found')

        return {
            '_type': 'url_transparent',
            'title': self._og_search_title(webpage),
            'url': video_url,
            'ie_key': ie.ie_key(),
        }


code:
from .common import InfoExtractor

class UKTVPlayIE(InfoExtractor):
    _VALID_URL = r'https?://uktvplay\.(?:uktv\.)?co\.uk/(?:.+?\?.*?\bvideo=|([^/]+/)*watch-online/)(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://uktvplay.uktv.co.uk/shows/world-at-war/c/200/watch-online/?video=2117008346001',
        'info_dict': {
            'id': '2117008346001',
            'ext': 'mp4',
            'title': 'Pincers',
            'description': 'Pincers',
            'uploader_id': '1242911124001',
            'upload_date': '20130124',
            'timestamp': 1359049267,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'expected_warnings': ['Failed to download MPD manifest']
    }, {
        'url': 'https://uktvplay.uktv.co.uk/shows/africa/watch-online/5983349675001',
        'only_matching': True,
    }]
    # BRIGHTCOVE_URL_TEMPLATE = 'https://players.brightcove.net/1242911124001/OrCyvJ2gyL_default/index.html?videoId=%s'
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1242911124001/H1xnMOqP_default/index.html?videoId=%s'

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return self.url_result(
            self.BRIGHTCOVE_URL_TEMPLATE % video_id,
            'BrightcoveNew', video_id)