code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys
import unittest
import re

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import copy
import json
import urllib.error

from test.helper import FakeYDL, assertRegexpMatches
from yt_dlp import YoutubeDL
from yt_dlp.compat import compat_os_name
from yt_dlp.extractor import YoutubeIE
from yt_dlp.extractor.common import InfoExtractor
from yt_dlp.postprocessor.common import PostProcessor
from yt_dlp.utils import (
    ExtractorError,
    LazyList,
    OnDemandPagedList,
    int_or_none,
    match_filter_func,
)

TEST_URL = 'http://localhost/sample.mp4'

class YDL(FakeYDL):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.downloaded_info_dicts = []
        self.msgs = []

    def process_info(self, info_dict):
        self.downloaded_info_dicts.append(info_dict.copy())

    def to_screen(self, msg, *args, **kwargs):
        self.msgs.append(msg)

    def dl(self, *args, **kwargs):
        assert False, 'Downloader must not be invoked for test_YoutubeDL'

def _make_result(formats, **kwargs):
    res = {
        'formats': formats,
        'id': 'testid',
        'title': 'testttitle',
        'extractor': 'testex',
        'extractor_key': 'TestEx',
        'webpage_url': 'http://example.com/watch?v=shenanigans',
    }
    res.update(**kwargs)
    return res

class TestFormatSelection(unittest.TestCase):
    def test_prefer_free_formats(self):
        # Same resolution => download webm
        ydl = YDL()
        ydl.params['prefer_free_formats'] = True
        formats = [
            {'ext': 'webm', 'height': 460, 'url': TEST_URL},
            {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['ext'], 'webm')

        # Different resolution => download best quality (mp4)
        ydl = YDL()
        ydl.params['prefer_free_formats'] = True
        formats = [
            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
            {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
        ]
        info_dict['formats'] = formats
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['ext'], 'mp4')

        # No prefer_free_formats => prefer mp4 and webm
        ydl = YDL()
        ydl.params['prefer_free_formats'] = False
        formats = [
            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
            {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
            {'ext': 'flv', 'height': 720, 'url': TEST_URL},
        ]
        info_dict['formats'] = formats
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['ext'], 'mp4')

        ydl = YDL()
        ydl.params['prefer_free_formats'] = False
        formats = [
            {'ext': 'flv', 'height': 720, 'url': TEST_URL},
            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
        ]
        info_dict['formats'] = formats
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['ext'], 'webm')

    def test_format_selection(self):
        formats = [
            {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
            {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
            {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
            {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
            {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        def test(inp, *expected, multi=False):
            ydl = YDL({
                'format': inp,
                'allow_multiple_video_streams': multi,
                'allow_multiple_audio_streams': multi,
            })
            ydl.process_ie_result(info_dict.copy())
            downloaded = map(lambda x: x['format_id'], ydl.downloaded_info_dicts)
            self.assertEqual(list(downloaded), list(expected))

        test('20/47', '47')
        test('20/71/worst', '35')
        test(None, '2')
        test('webm/mp4', '47')
        test('3gp/40/mp4', '35')
        test('example-with-dashes', 'example-with-dashes')
        test('all', '2', '47', '45', 'example-with-dashes', '35')
        test('mergeall', '2+47+45+example-with-dashes+35', multi=True)

    def test_format_selection_audio(self):
        formats = [
            {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
            {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
            {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
            {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'bestaudio'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'audio-high')

        ydl = YDL({'format': 'worstaudio'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'audio-low')

        formats = [
            {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
            {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'bestaudio/worstaudio/best'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'vid-high')

    def test_format_selection_audio_exts(self):
        formats = [
            {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
            {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
            {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
            {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
            {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
        ]

        info_dict = _make_result(formats)
        ydl = YDL({'format': 'best'})
        ie = YoutubeIE(ydl)
        ie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(copy.deepcopy(info_dict))
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'aac-64')

        ydl = YDL({'format': 'mp3'})
        ie = YoutubeIE(ydl)
        ie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(copy.deepcopy(info_dict))
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'mp3-64')

        ydl = YDL({'prefer_free_formats': True})
        ie = YoutubeIE(ydl)
        ie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(copy.deepcopy(info_dict))
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'ogg-64')

    def test_format_selection_video(self):
        formats = [
            {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
            {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
            {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'bestvideo'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'dash-video-high')

        ydl = YDL({'format': 'worstvideo'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'dash-video-low')

        ydl = YDL({'format': 'bestvideo[format_id^=dash][format_id$=low]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'dash-video-low')

        formats = [
            {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')

    def test_format_selection_string_ops(self):
        formats = [
            {'format_id': 'abc-cba', 'ext': 'mp4', 'url': TEST_URL},
            {'format_id': 'zxc-cxz', 'ext': 'webm', 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        # equals (=)
        ydl = YDL({'format': '[format_id=abc-cba]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'abc-cba')

        # does not equal (!=)
        ydl = YDL({'format': '[format_id!=abc-cba]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'zxc-cxz')

        ydl = YDL({'format': '[format_id!=abc-cba][format_id!=zxc-cxz]'})
        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())

        # starts with (^=)
        ydl = YDL({'format': '[format_id^=abc]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'abc-cba')

        # does not start with (!^=)
        ydl = YDL({'format': '[format_id!^=abc]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'zxc-cxz')

        ydl = YDL({'format': '[format_id!^=abc][format_id!^=zxc]'})
        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())

        # ends with ($=)
        ydl = YDL({'format': '[format_id$=cba]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'abc-cba')

        # does not end with (!$=)
        ydl = YDL({'format': '[format_id!$=cba]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'zxc-cxz')

        ydl = YDL({'format': '[format_id!$=cba][format_id!$=cxz]'})
        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())

        # contains (*=)
        ydl = YDL({'format': '[format_id*=bc-cb]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'abc-cba')

        # does not contain (!*=)
        ydl = YDL({'format': '[format_id!*=bc-cb]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'zxc-cxz')

        ydl = YDL({'format': '[format_id!*=abc][format_id!*=zxc]'})
        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())

        ydl = YDL({'format': '[format_id!*=-]'})
        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())

    def test_youtube_format_selection(self):
        # FIXME: Rewrite in accordance with the new format sorting options
        return

        order = [
            '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
            # Apple HTTP Live Streaming
            '96', '95', '94', '93', '92', '132', '151',
            # 3D
            '85', '84', '102', '83', '101', '82', '100',
            # Dash video
            '137', '248', '136', '247', '135', '246',
            '245', '244', '134', '243', '133', '242', '160',
            # Dash audio
            '141', '172', '140', '171', '139',
        ]

        def format_info(f_id):
            info = YoutubeIE._formats[f_id].copy()

            # XXX: In real cases InfoExtractor._parse_mpd_formats() fills up 'acodec'
            # and 'vcodec', while in tests such information is incomplete since
            # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
            # test_YoutubeDL.test_youtube_format_selection is broken without
            # this fix
            if 'acodec' in info and 'vcodec' not in info:
                info['vcodec'] = 'none'
            elif 'vcodec' in info and 'acodec' not in info:
                info['acodec'] = 'none'

            info['format_id'] = f_id
            info['url'] = 'url:' + f_id
            return info
        formats_order = [format_info(f_id) for f_id in order]

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': 'bestvideo+bestaudio'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], '248+172')
        self.assertEqual(downloaded['ext'], 'mp4')

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], '38')

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': 'bestvideo/best,bestaudio'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
        self.assertEqual(downloaded_ids, ['137', '141'])

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
        self.assertEqual(downloaded_ids, ['137+141', '248+141'])

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
        self.assertEqual(downloaded_ids, ['136+141', '247+141'])

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
        self.assertEqual(downloaded_ids, ['248+141'])

        for f1, f2 in zip(formats_order, formats_order[1:]):
            info_dict = _make_result([f1, f2], extractor='youtube')
            ydl = YDL({'format': 'best/bestvideo'})
            yie = YoutubeIE(ydl)
            yie._sort_formats(info_dict['formats'])
            ydl.process_ie_result(info_dict)
            downloaded = ydl.downloaded_info_dicts[0]
            self.assertEqual(downloaded['format_id'], f1['format_id'])

            info_dict = _make_result([f2, f1], extractor='youtube')
            ydl = YDL({'format': 'best/bestvideo'})
            yie = YoutubeIE(ydl)
            yie._sort_formats(info_dict['formats'])
            ydl.process_ie_result(info_dict)
            downloaded = ydl.downloaded_info_dicts[0]
            self.assertEqual(downloaded['format_id'], f1['format_id'])

    def test_audio_only_extractor_format_selection(self):
        # For extractors with incomplete formats (all formats are audio-only or
        # video-only) best and worst should fallback to corresponding best/worst
        # video-only or audio-only formats (as per
        # https://github.com/ytdl-org/youtube-dl/pull/5556)
        formats = [
            {'format_id': 'low', 'ext': 'mp3', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
            {'format_id': 'high', 'ext': 'mp3', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'best'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'high')

        ydl = YDL({'format': 'worst'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'low')

    def test_format_not_available(self):
        formats = [
            {'format_id': 'regular', 'ext': 'mp4', 'height': 360, 'url': TEST_URL},
            {'format_id': 'video', 'ext': 'mp4', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        # This must fail since complete video-audio format does not match filter
        # and extractor does not provide incomplete only formats (i.e. only
        # video-only or audio-only).
        ydl = YDL({'format': 'best[height>360]'})
        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())

    def test_format_selection_issue_10083(self):
        # See https://github.com/ytdl-org/youtube-dl/issues/10083
        formats = [
            {'format_id': 'regular', 'height': 360, 'url': TEST_URL},
            {'format_id': 'video', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
            {'format_id': 'audio', 'vcodec': 'none', 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'best[height>360]/bestvideo[height>360]+bestaudio'})
        ydl.process_ie_result(info_dict.copy())
        self.assertEqual(ydl.downloaded_info_dicts[0]['format_id'], 'video+audio')

    def test_invalid_format_specs(self):
        def assert_syntax_error(format_spec):
            self.assertRaises(SyntaxError, YDL, {'format': format_spec})

        assert_syntax_error('bestvideo,,best')
        assert_syntax_error('+bestaudio')
        assert_syntax_error('bestvideo+')
        assert_syntax_error('/')
        assert_syntax_error('[720<height]')

    def test_format_filtering(self):
        formats = [
            {'format_id': 'A', 'filesize': 500, 'width': 1000},
            {'format_id': 'B', 'filesize': 1000, 'width': 500},
            {'format_id': 'C', 'filesize': 1000, 'width': 400},
            {'format_id': 'D', 'filesize': 2000, 'width': 600},
            {'format_id': 'E', 'filesize': 3000},
            {'format_id': 'F'},
            {'format_id': 'G', 'filesize': 1000000},
        ]
        for f in formats:
            f['url'] = 'http://_/'
            f['ext'] = 'unknown'
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'best[filesize<3000]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'D')

        ydl = YDL({'format': 'best[filesize<=3000]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'E')

        ydl = YDL({'format': 'best[filesize <= ? 3000]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'F')

        ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'B')

        ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'C')

        ydl = YDL({'format': '[filesize>?1]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'G')

        ydl = YDL({'format': '[filesize<1M]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'E')

        ydl = YDL({'format': '[filesize<1MiB]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'G')

        ydl = YDL({'format': 'all[width>=400][width<=600]'})
        ydl.process_ie_result(info_dict)
        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
        self.assertEqual(downloaded_ids, ['D', 'C', 'B'])

        ydl = YDL({'format': 'best[height<40]'})
        try:
            ydl.process_ie_result(info_dict)
        except ExtractorError:
            pass
        self.assertEqual(ydl.downloaded_info_dicts, [])

    def test_default_format_spec(self):
        ydl = YDL({'simulate': True})
        self.assertEqual(ydl._default_format_spec({}), 'bestvideo*+bestaudio/best')

        ydl = YDL({})
        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')

        ydl = YDL({'simulate': True})
        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'bestvideo*+bestaudio/best')

        ydl = YDL({'outtmpl': '-'})
        self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')

        ydl = YDL({})
        self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo*+bestaudio/best')
        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')

class TestYoutubeDL(unittest.TestCase):
    def test_subtitles(self):
        def s_formats(lang, autocaption=False):
            return [{
                'ext': ext,
                'url': f'http://localhost/video.{lang}.{ext}',
                '_auto': autocaption,
            } for ext in ['vtt', 'srt', 'ass']]
        subtitles = {l: s_formats(l) for l in ['en', 'fr', 'es']}
        auto_captions = {l: s_formats(l, True) for l in ['it', 'pt', 'es']}
        info_dict = {
            'id': 'test',
            'title': 'Test',
            'url': 'http://localhost/video.mp4',
            'subtitles': subtitles,
            'automatic_captions': auto_captions,
            'extractor': 'TEST',
            'webpage_url': 'http://example.com/watch?v=shenanigans',
        }

        def get_info(params={}):
            params.setdefault('simulate', True)
            ydl = YDL(params)
            ydl.report_warning = lambda *args, **kargs: None
            return ydl.process_video_result(info_dict, download=False)

        result = get_info()
        self.assertFalse(result.get('requested_subtitles'))
        self.assertEqual(result['subtitles'], subtitles)
        self.assertEqual(result['automatic_captions'], auto_captions)

        result = get_info({'writesubtitles': True})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), {'en'})
        self.assertTrue(subs['en'].get('data') is None)
        self.assertEqual(subs['en']['ext'], 'ass')

        result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
        subs = result['requested_subtitles']
        self.assertEqual(subs['en']['ext'], 'srt')

        result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), {'es', 'fr'})

        result = get_info({'writesubtitles': True, 'subtitleslangs': ['all', '-en']})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), {'es', 'fr'})

        result = get_info({'writesubtitles': True, 'subtitleslangs': ['en', 'fr', '-en']})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), {'fr'})

        result = get_info({'writesubtitles': True, 'subtitleslangs': ['-en', 'en']})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), {'en'})

        result = get_info({'writesubtitles': True, 'subtitleslangs': ['e.+']})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), {'es', 'en'})

        result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), {'es', 'pt'})
        self.assertFalse(subs['es']['_auto'])
        self.assertTrue(subs['pt']['_auto'])

        result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), {'es', 'pt'})
        self.assertTrue(subs['es']['_auto'])
        self.assertTrue(subs['pt']['_auto'])

    def test_add_extra_info(self):
        test_dict = {
            'extractor': 'Foo',
        }
        extra_info = {
            'extractor': 'Bar',
            'playlist': 'funny videos',
        }
        YDL.add_extra_info(test_dict, extra_info)
        self.assertEqual(test_dict['extractor'], 'Foo')
        self.assertEqual(test_dict['playlist'], 'funny videos')

    outtmpl_info = {
        'id': '1234',
        'ext': 'mp4',
        'width': None,
        'height': 1080,
        'filesize': 1024,
        'title1': '$PATH',
        'title2': '%PATH%',
        'title3': 'foo/bar\\test',
        'title4': 'foo "bar" test',
        'title5': 'áéí 𝐀',
        'timestamp': 1618488000,
        'duration': 100000,
        'playlist_index': 1,
        'playlist_autonumber': 2,
        '__last_playlist_index': 100,
        'n_entries': 10,
        'formats': [
            {'id': 'id 1', 'height': 1080, 'width': 1920},
            {'id': 'id 2', 'height': 720},
            {'id': 'id 3'}
        ]
    }

    def test_prepare_outtmpl_and_filename(self):
        def test(tmpl, expected, *, info=None, **params):
            params['outtmpl'] = tmpl
            ydl = FakeYDL(params)
            ydl._num_downloads = 1
            self.assertEqual(ydl.validate_outtmpl(tmpl), None)

            out = ydl.evaluate_outtmpl(tmpl, info or self.outtmpl_info)
            fname = ydl.prepare_filename(info or self.outtmpl_info)

            if not isinstance(expected, (list, tuple)):
                expected = (expected, expected)
            for (name, got), expect in zip((('outtmpl', out), ('filename', fname)), expected):
                if callable(expect):
                    self.assertTrue(expect(got), f'Wrong {name} from {tmpl}')
                else:
                    self.assertEqual(got, expect, f'Wrong {name} from {tmpl}')

        # Side-effects
        original_infodict = dict(self.outtmpl_info)
        test('foo.bar', 'foo.bar')
        original_infodict['epoch'] = self.outtmpl_info.get('epoch')
        self.assertTrue(isinstance(original_infodict['epoch'], int))
        test('%(epoch)d', int_or_none)
        self.assertEqual(original_infodict, self.outtmpl_info)

        # Auto-generated fields
        test('%(id)s.%(ext)s', '1234.mp4')
        test('%(duration_string)s', ('27:46:40', '27-46-40'))
        test('%(resolution)s', '1080p')
        test('%(playlist_index)s', '001')
        test('%(playlist_autonumber)s', '02')
        test('%(autonumber)s', '00001')
        test('%(autonumber+2)03d', '005', autonumber_start=3)
        test('%(autonumber)s', '001', autonumber_size=3)

        # Escaping %
        test('%', '%')
        test('%%', '%')
        test('%%%%', '%%')
        test('%s', '%s')
        test('%%%s', '%%s')
        test('%d', '%d')
        test('%abc%', '%abc%')
        test('%%(width)06d.%(ext)s', '%(width)06d.mp4')
        test('%%%(height)s', '%1080')
        test('%(width)06d.%(ext)s', 'NA.mp4')
        test('%(width)06d.%%(ext)s', 'NA.%(ext)s')
        test('%%(width)06d.%(ext)s', '%(width)06d.mp4')

        # ID sanitization
        test('%(id)s', '_abcd', info={'id': '_abcd'})
        test('%(some_id)s', '_abcd', info={'some_id': '_abcd'})
        test('%(formats.0.id)s', '_abcd', info={'formats': [{'id': '_abcd'}]})
        test('%(id)s', '-abcd', info={'id': '-abcd'})
        test('%(id)s', '.abcd', info={'id': '.abcd'})
        test('%(id)s', 'ab__cd', info={'id': 'ab__cd'})
        test('%(id)s', ('ab:cd', 'ab -cd'), info={'id': 'ab:cd'})
        test('%(id.0)s', '-', info={'id': '--'})

        # Invalid templates
        self.assertTrue(isinstance(YoutubeDL.validate_outtmpl('%(title)'), ValueError))
        test('%(invalid@tmpl|def)s', 'none', outtmpl_na_placeholder='none')
        test('%(..)s', 'NA')
        test('%(formats.{id)s', 'NA')

        # Entire info_dict
        def expect_same_infodict(out):
            got_dict = json.loads(out)
            for info_field, expected in self.outtmpl_info.items():
                self.assertEqual(got_dict.get(info_field), expected, info_field)
            return True

        test('%()j', (expect_same_infodict, str))

        # NA placeholder
        NA_TEST_OUTTMPL = '%(uploader_date)s-%(width)d-%(x|def)s-%(id)s.%(ext)s'
        test(NA_TEST_OUTTMPL, 'NA-NA-def-1234.mp4')
        test(NA_TEST_OUTTMPL, 'none-none-def-1234.mp4', outtmpl_na_placeholder='none')
        test(NA_TEST_OUTTMPL, '--def-1234.mp4', outtmpl_na_placeholder='')
        test('%(non_existent.0)s', 'NA')

        # String formatting
        FMT_TEST_OUTTMPL = '%%(height)%s.%%(ext)s'
        test(FMT_TEST_OUTTMPL % 's', '1080.mp4')
        test(FMT_TEST_OUTTMPL % 'd', '1080.mp4')
        test(FMT_TEST_OUTTMPL % '6d', '  1080.mp4')
        test(FMT_TEST_OUTTMPL % '-6d', '1080  .mp4')
        test(FMT_TEST_OUTTMPL % '06d', '001080.mp4')
        test(FMT_TEST_OUTTMPL % ' 06d', ' 01080.mp4')
        test(FMT_TEST_OUTTMPL % '   06d', ' 01080.mp4')
        test(FMT_TEST_OUTTMPL % '0 6d', ' 01080.mp4')
        test(FMT_TEST_OUTTMPL % '0   6d', ' 01080.mp4')
        test(FMT_TEST_OUTTMPL % '   0   6d', ' 01080.mp4')

        # Type casting
        test('%(id)d', '1234')
        test('%(height)c', '1')
        test('%(ext)c', 'm')
        test('%(id)d %(id)r', "1234 '1234'")
        test('%(id)r %(height)r', "'1234' 1080")
        test('%(ext)s-%(ext|def)d', 'mp4-def')
        test('%(width|0)04d', '0000')
        test('a%(width|)d', 'a', outtmpl_na_placeholder='none')

        FORMATS = self.outtmpl_info['formats']
        sanitize = lambda x: x.replace(':', ' -').replace('"', "'").replace('\n', ' ')

        # Custom type casting
        test('%(formats.:.id)l', 'id 1, id 2, id 3')
        test('%(formats.:.id)#l', ('id 1\nid 2\nid 3', 'id 1 id 2 id 3'))
        test('%(ext)l', 'mp4')
        test('%(formats.:.id) 18l', '  id 1, id 2, id 3')
        test('%(formats)j', (json.dumps(FORMATS), sanitize(json.dumps(FORMATS))))
        test('%(formats)#j', (json.dumps(FORMATS, indent=4), sanitize(json.dumps(FORMATS, indent=4))))
        test('%(title5).3B', 'á')
        test('%(title5)U', 'áéí 𝐀')
        test('%(title5)#U', 'a\u0301e\u0301i\u0301 𝐀')
        test('%(title5)+U', 'áéí A')
        test('%(title5)+#U', 'a\u0301e\u0301i\u0301 A')
        test('%(height)D', '1k')
        test('%(filesize)#D', '1Ki')
        test('%(height)5.2D', ' 1.08k')
        test('%(title4)#S', 'foo_bar_test')
        test('%(title4).10S', ('foo \'bar\' ', 'foo \'bar\'' + ('#' if compat_os_name == 'nt' else ' ')))
        if compat_os_name == 'nt':
            test('%(title4)q', ('"foo \\"bar\\" test"', "'foo _'bar_' test'"))
            test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', "'id 1' 'id 2' 'id 3'"))
            test('%(formats.0.id)#q', ('"id 1"', "'id 1'"))
        else:
            test('%(title4)q', ('\'foo "bar" test\'', "'foo 'bar' test'"))
            test('%(formats.:.id)#q', "'id 1' 'id 2' 'id 3'")
            test('%(formats.0.id)#q', "'id 1'")

        # Internal formatting
        test('%(timestamp-1000>%H-%M-%S)s', '11-43-20')
        test('%(title|%)s %(title|%%)s', '% %%')
        test('%(id+1-height+3)05d', '00158')
        test('%(width+100)05d', 'NA')
        test('%(formats.0) 15s', ('% 15s' % FORMATS[0], '% 15s' % sanitize(str(FORMATS[0]))))
        test('%(formats.0)r', (repr(FORMATS[0]), sanitize(repr(FORMATS[0]))))
        test('%(height.0)03d', '001')
        test('%(-height.0)04d', '-001')
        test('%(formats.-1.id)s', FORMATS[-1]['id'])
        test('%(formats.0.id.-1)d', FORMATS[0]['id'][-1])
        test('%(formats.3)s', 'NA')
        test('%(formats.:2:-1)r', repr(FORMATS[:2:-1]))
        test('%(formats.0.id.-1+id)f', '1235.000000')
        test('%(formats.0.id.-1+formats.1.id.-1)d', '3')
        out = json.dumps([{'id': f['id'], 'height.:2': str(f['height'])[:2]}
                          if 'height' in f else {'id': f['id']}
                          for f in FORMATS])
        test('%(formats.:.{id,height.:2})j', (out, sanitize(out)))
        test('%(formats.:.{id,height}.id)l', ', '.join(f['id'] for f in FORMATS))
        test('%(.{id,title})j', ('{"id": "1234"}', "{'id' - '1234'}"))

        # Alternates
        test('%(title,id)s', '1234')
        test('%(width-100,height+20|def)d', '1100')
        test('%(width-100,height+width|def)s', 'def')
        test('%(timestamp-x>%H\\,%M\\,%S,timestamp>%H\\,%M\\,%S)s', '12,00,00')

        # Replacement
        test('%(id&foo)s.bar', 'foo.bar')
        test('%(title&foo)s.bar', 'NA.bar')
        test('%(title&foo|baz)s.bar', 'baz.bar')
        test('%(x,id&foo|baz)s.bar', 'foo.bar')
        test('%(x,title&foo|baz)s.bar', 'baz.bar')

        # Laziness
        def gen():
            yield from range(5)
            raise self.assertTrue(False, 'LazyList should not be evaluated till here')
        test('%(key.4)s', '4', info={'key': LazyList(gen())})

        # Empty filename
        test('%(foo|)s-%(bar|)s.%(ext)s', '-.mp4')
        # test('%(foo|)s.%(ext)s', ('.mp4', '_.mp4'))  # fixme
        # test('%(foo|)s', ('', '_'))  # fixme

        # Environment variable expansion for prepare_filename
        os.environ['__yt_dlp_var'] = 'expanded'
        envvar = '%__yt_dlp_var%' if compat_os_name == 'nt' else '$__yt_dlp_var'
        test(envvar, (envvar, 'expanded'))
        if compat_os_name == 'nt':
            test('%s%', ('%s%', '%s%'))
            os.environ['s'] = 'expanded'
            test('%s%', ('%s%', 'expanded'))  # %s% should be expanded before escaping %s
            os.environ['(test)s'] = 'expanded'
            test('%(test)s%', ('NA%', 'expanded'))  # Environment should take priority over template

        # Path expansion and escaping
        test('Hello %(title1)s', 'Hello $PATH')
        test('Hello %(title2)s', 'Hello %PATH%')
        test('%(title3)s', ('foo/bar\\test', 'foo_bar_test'))
        test('folder/%(title3)s', ('folder/foo/bar\\test', 'folder%sfoo_bar_test' % os.path.sep))

    def test_format_note(self):
        ydl = YoutubeDL()
        self.assertEqual(ydl._format_note({}), '')
        assertRegexpMatches(self, ydl._format_note({
            'vbr': 10,
        }), r'^\s*10k$')
        assertRegexpMatches(self, ydl._format_note({
            'fps': 30,
        }), r'^30fps$')

    def test_postprocessors(self):
        filename = 'post-processor-testfile.mp4'
        audiofile = filename + '.mp3'

        class SimplePP(PostProcessor):
            def run(self, info):
                with open(audiofile, 'wt') as f:
                    f.write('EXAMPLE')
                return [info['filepath']], info

        def run_pp(params, PP):
            with open(filename, 'wt') as f:
                f.write('EXAMPLE')
            ydl = YoutubeDL(params)
            ydl.add_post_processor(PP())
            ydl.post_process(filename, {'filepath': filename})

        run_pp({'keepvideo': True}, SimplePP)
        self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
        self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
        os.unlink(filename)
        os.unlink(audiofile)

        run_pp({'keepvideo': False}, SimplePP)
        self.assertFalse(os.path.exists(filename), '%s exists' % filename)
        self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
        os.unlink(audiofile)

        class ModifierPP(PostProcessor):
            def run(self, info):
                with open(info['filepath'], 'wt') as f:
                    f.write('MODIFIED')
                return [], info

        run_pp({'keepvideo': False}, ModifierPP)
        self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
        os.unlink(filename)

    def test_match_filter(self):
        first = {
            'id': '1',
            'url': TEST_URL,
            'title': 'one',
            'extractor': 'TEST',
            'duration': 30,
            'filesize': 10 * 1024,
            'playlist_id': '42',
            'uploader': "變態妍字幕版 太妍 тест",
            'creator': "тест ' 123 ' тест--",
            'webpage_url': 'http://example.com/watch?v=shenanigans',
        }
        second = {
            'id': '2',
            'url': TEST_URL,
            'title': 'two',
            'extractor': 'TEST',
            'duration': 10,
            'description': 'foo',
            'filesize': 5 * 1024,
            'playlist_id': '43',
            'uploader': "тест 123",
            'webpage_url': 'http://example.com/watch?v=SHENANIGANS',
        }
        videos = [first, second]

        def get_videos(filter_=None):
            ydl = YDL({'match_filter': filter_, 'simulate': True})
            for v in videos:
                ydl.process_ie_result(v, download=True)
            return [v['id'] for v in ydl.downloaded_info_dicts]

        res = get_videos()
        self.assertEqual(res, ['1', '2'])

        def f(v, incomplete):
            if v['id'] == '1':
                return None
            else:
                return 'Video id is not 1'
        res = get_videos(f)
        self.assertEqual(res, ['1'])

        f = match_filter_func('duration < 30')
        res = get_videos(f)
        self.assertEqual(res, ['2'])

        f = match_filter_func('description = foo')
        res = get_videos(f)
        self.assertEqual(res, ['2'])

        f = match_filter_func('description =? foo')
        res = get_videos(f)
        self.assertEqual(res, ['1', '2'])

        f = match_filter_func('filesize > 5KiB')
        res = get_videos(f)
        self.assertEqual(res, ['1'])

        f = match_filter_func('playlist_id = 42')
        res = get_videos(f)
        self.assertEqual(res, ['1'])

        f = match_filter_func('uploader = "變態妍字幕版 太妍 тест"')
        res = get_videos(f)
        self.assertEqual(res, ['1'])

        f = match_filter_func('uploader != "變態妍字幕版 太妍 тест"')
        res = get_videos(f)
        self.assertEqual(res, ['2'])

        f = match_filter_func('creator = "тест \' 123 \' тест--"')
        res = get_videos(f)
        self.assertEqual(res, ['1'])

        f = match_filter_func("creator = 'тест \\' 123 \\' тест--'")
        res = get_videos(f)
        self.assertEqual(res, ['1'])

        f = match_filter_func(r"creator = 'тест \' 123 \' тест--' & duration > 30")
        res = get_videos(f)
        self.assertEqual(res, [])

    def test_playlist_items_selection(self):
        INDICES, PAGE_SIZE = list(range(1, 11)), 3

        def entry(i, evaluated):
            evaluated.append(i)
            return {
                'id': str(i),
                'title': str(i),
                'url': TEST_URL,
            }

        def pagedlist_entries(evaluated):
            def page_func(n):
                start = PAGE_SIZE * n
                for i in INDICES[start: start + PAGE_SIZE]:
                    yield entry(i, evaluated)
            return OnDemandPagedList(page_func, PAGE_SIZE)

        def page_num(i):
            return (i + PAGE_SIZE - 1) // PAGE_SIZE

        def generator_entries(evaluated):
            for i in INDICES:
                yield entry(i, evaluated)

        def list_entries(evaluated):
            return list(generator_entries(evaluated))

        def lazylist_entries(evaluated):
            return LazyList(generator_entries(evaluated))

        def get_downloaded_info_dicts(params, entries):
            ydl = YDL(params)
            ydl.process_ie_result({
                '_type': 'playlist',
                'id': 'test',
                'extractor': 'test:playlist',
                'extractor_key': 'test:playlist',
                'webpage_url': 'http://example.com',
                'entries': entries,
            })
            return ydl.downloaded_info_dicts

        def test_selection(params, expected_ids, evaluate_all=False):
            expected_ids = list(expected_ids)
            if evaluate_all:
                generator_eval = pagedlist_eval = INDICES
            elif not expected_ids:
                generator_eval = pagedlist_eval = []
            else:
                generator_eval = INDICES[0: max(expected_ids)]
                pagedlist_eval = INDICES[PAGE_SIZE * page_num(min(expected_ids)) - PAGE_SIZE:
                                         PAGE_SIZE * page_num(max(expected_ids))]

            for name, func, expected_eval in (
                ('list', list_entries, INDICES),
                ('Generator', generator_entries, generator_eval),
                # ('LazyList', lazylist_entries, generator_eval),  # Generator and LazyList follow the exact same code path
                ('PagedList', pagedlist_entries, pagedlist_eval),
            ):
                evaluated = []
                entries = func(evaluated)
                results = [(v['playlist_autonumber'] - 1, (int(v['id']), v['playlist_index']))
                           for v in get_downloaded_info_dicts(params, entries)]
                self.assertEqual(results, list(enumerate(zip(expected_ids, expected_ids))), f'Entries of {name} for {params}')
                self.assertEqual(sorted(evaluated), expected_eval, f'Evaluation of {name} for {params}')

        test_selection({}, INDICES)
        test_selection({'playlistend': 20}, INDICES, True)
        test_selection({'playlistend': 2}, INDICES[:2])
        test_selection({'playliststart': 11}, [], True)
        test_selection({'playliststart': 2}, INDICES[1:])
        test_selection({'playlist_items': '2-4'}, INDICES[1:4])
        test_selection({'playlist_items': '2,4'}, [2, 4])
        test_selection({'playlist_items': '20'}, [], True)
        test_selection({'playlist_items': '0'}, [])

        # Tests for https://github.com/ytdl-org/youtube-dl/issues/10591
        test_selection({'playlist_items': '2-4,3-4,3'}, [2, 3, 4])
        test_selection({'playlist_items': '4,2'}, [4, 2])

        # Tests for https://github.com/yt-dlp/yt-dlp/issues/720
        # https://github.com/yt-dlp/yt-dlp/issues/302
        test_selection({'playlistreverse': True}, INDICES[::-1])
        test_selection({'playliststart': 2, 'playlistreverse': True}, INDICES[:0:-1])
        test_selection({'playlist_items': '2,4', 'playlistreverse': True}, [4, 2])
        test_selection({'playlist_items': '4,2'}, [4, 2])

        # Tests for --playlist-items start:end:step
        test_selection({'playlist_items': ':'}, INDICES, True)
        test_selection({'playlist_items': '::1'}, INDICES, True)
        test_selection({'playlist_items': '::-1'}, INDICES[::-1], True)
        test_selection({'playlist_items': ':6'}, INDICES[:6])
        test_selection({'playlist_items': ':-6'}, INDICES[:-5], True)
        test_selection({'playlist_items': '-1:6:-2'}, INDICES[:4:-2], True)
        test_selection({'playlist_items': '9:-6:-2'}, INDICES[8:3:-2], True)

        test_selection({'playlist_items': '1:inf:2'}, INDICES[::2], True)
        test_selection({'playlist_items': '-2:inf'}, INDICES[-2:], True)
        test_selection({'playlist_items': ':inf:-1'}, [], True)
        test_selection({'playlist_items': '0-2:2'}, [2])
        test_selection({'playlist_items': '1-:2'}, INDICES[::2], True)
        test_selection({'playlist_items': '0--2:2'}, INDICES[1:-1:2], True)

        test_selection({'playlist_items': '10::3'}, [10], True)
        test_selection({'playlist_items': '-1::3'}, [10], True)
        test_selection({'playlist_items': '11::3'}, [], True)
        test_selection({'playlist_items': '-15::2'}, INDICES[1::2], True)
        test_selection({'playlist_items': '-15::15'}, [], True)

    def test_urlopen_no_file_protocol(self):
        # see https://github.com/ytdl-org/youtube-dl/issues/8227
        ydl = YDL()
        self.assertRaises(urllib.error.URLError, ydl.urlopen, 'file:///etc/passwd')

    def test_do_not_override_ie_key_in_url_transparent(self):
        ydl = YDL()

        class Foo1IE(InfoExtractor):
            _VALID_URL = r'foo1:'

            def _real_extract(self, url):
                return {
                    '_type': 'url_transparent',
                    'url': 'foo2:',
                    'ie_key': 'Foo2',
                    'title': 'foo1 title',
                    'id': 'foo1_id',
                }

        class Foo2IE(InfoExtractor):
            _VALID_URL = r'foo2:'

            def _real_extract(self, url):
                return {
                    '_type': 'url',
                    'url': 'foo3:',
                    'ie_key': 'Foo3',
                }

        class Foo3IE(InfoExtractor):
            _VALID_URL = r'foo3:'

            def _real_extract(self, url):
                return _make_result([{'url': TEST_URL}], title='foo3 title')

        ydl.add_info_extractor(Foo1IE(ydl))
        ydl.add_info_extractor(Foo2IE(ydl))
        ydl.add_info_extractor(Foo3IE(ydl))
        ydl.extract_info('foo1:')
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['url'], TEST_URL)
        self.assertEqual(downloaded['title'], 'foo1 title')
        self.assertEqual(downloaded['id'], 'testid')
        self.assertEqual(downloaded['extractor'], 'testex')
        self.assertEqual(downloaded['extractor_key'], 'TestEx')

    # Test case for https://github.com/ytdl-org/youtube-dl/issues/27064
    def test_ignoreerrors_for_playlist_with_url_transparent_iterable_entries(self):

        class _YDL(YDL):
            def __init__(self, *args, **kwargs):
                super().__init__(*args, **kwargs)

            def trouble(self, s, tb=None):
                pass

        ydl = _YDL({
            'format': 'extra',
            'ignoreerrors': True,
        })

        class VideoIE(InfoExtractor):
            _VALID_URL = r'video:(?P<id>\d+)'

            def _real_extract(self, url):
                video_id = self._match_id(url)
                formats = [{
                    'format_id': 'default',
                    'url': 'url:',
                }]
                if video_id == '0':
                    raise ExtractorError('foo')
                if video_id == '2':
                    formats.append({
                        'format_id': 'extra',
                        'url': TEST_URL,
                    })
                return {
                    'id': video_id,
                    'title': 'Video %s' % video_id,
                    'formats': formats,
                }

        class PlaylistIE(InfoExtractor):
            _VALID_URL = r'playlist:'

            def _entries(self):
                for n in range(3):
                    video_id = str(n)
                    yield {
                        '_type': 'url_transparent',
                        'ie_key': VideoIE.ie_key(),
                        'id': video_id,
                        'url': 'video:%s' % video_id,
                        'title': 'Video Transparent %s' % video_id,
                    }

            def _real_extract(self, url):
                return self.playlist_result(self._entries())

        ydl.add_info_extractor(VideoIE(ydl))
        ydl.add_info_extractor(PlaylistIE(ydl))
        info = ydl.extract_info('playlist:')
        entries = info['entries']
        self.assertEqual(len(entries), 3)
        self.assertTrue(entries[0] is None)
        self.assertTrue(entries[1] is None)
        self.assertEqual(len(ydl.downloaded_info_dicts), 1)
        downloaded = ydl.downloaded_info_dicts[0]
        entries[2].pop('requested_downloads', None)
        self.assertEqual(entries[2], downloaded)
        self.assertEqual(downloaded['url'], TEST_URL)
        self.assertEqual(downloaded['title'], 'Video Transparent 2')
        self.assertEqual(downloaded['id'], '2')
        self.assertEqual(downloaded['extractor'], 'Video')
        self.assertEqual(downloaded['extractor_key'], 'Video')

class TestFilenameTest(unittest.TestCase):
    def test_filenames(self):
        class _YDL(YDL):
            def __init__(self, *args, **kwargs):
                super(_YDL, self).__init__(*args, **kwargs)

        ydl = _YDL({
            'test_filename': 'python3 test/testdata/fntest/exc.py {}',
        })

        self.assertTrue(ydl.test_filename_external("It's a small world after all"))
        self.assertTrue(ydl.test_filename_external("It's a small world"))
        self.assertTrue(ydl.test_filename_external("It's A small world"))

        self.assertFalse(ydl.test_filename_external(''))
        self.assertFalse(ydl.test_filename_external("Es gibt eine mond"))

    def test_empty_command(self):
        class _YDL(YDL):
            def __init__(self, *args, **kwargs):
                super(_YDL, self).__init__(*args, **kwargs)

        ydl = _YDL({
            'test_filename': '',
        })

        self.assertTrue(ydl.test_filename_external("It's a small world after all"))
        self.assertTrue(ydl.test_filename_external("It's a small world"))
        self.assertTrue(ydl.test_filename_external("It's A small world"))

        self.assertTrue(ydl.test_filename_external(''))
        self.assertTrue(ydl.test_filename_external("Es gibt eine mond"))

    def test_unset(self):
        class _YDL(YDL):
            def __init__(self, *args, **kwargs):
                super(_YDL, self).__init__(*args, **kwargs)

        ydl = _YDL({})

        self.assertTrue(ydl.test_filename_external("It's a small world after all"))
        self.assertTrue(ydl.test_filename_external("It's a small world"))
        self.assertTrue(ydl.test_filename_external("It's A small world"))

        self.assertTrue(ydl.test_filename_external(''))
        self.assertTrue(ydl.test_filename_external("Es gibt eine mond"))

    def test_regex_str(self):
        class _YDL(YDL):
            def __init__(self, *args, **kwargs):
                super(_YDL, self).__init__(*args, **kwargs)

        ydl = _YDL({
            'test_filename': r"re:(?i)It's a small world",
        })

        self.assertTrue(ydl.test_filename_external("It's a small world after all"))
        self.assertTrue(ydl.test_filename_external("It's a small world"))
        self.assertTrue(ydl.test_filename_external("It's A small world"))

        self.assertFalse(ydl.test_filename_external(''))
        self.assertFalse(ydl.test_filename_external("Es gibt eine mond"))

    def test_compiled_regex(self):
        class _YDL(YDL):
            def __init__(self, *args, **kwargs):
                super(_YDL, self).__init__(*args, **kwargs)

        ydl = _YDL({
            'test_filename': re.compile(r"(?i)It's a small world"),
        })

        self.assertTrue(ydl.test_filename_external("It's a small world after all"))
        self.assertTrue(ydl.test_filename_external("It's a small world"))
        self.assertTrue(ydl.test_filename_external("It's A small world"))

        self.assertFalse(ydl.test_filename_external(''))
        self.assertFalse(ydl.test_filename_external("Es gibt eine mond"))

if __name__ == '__main__':
    unittest.main()


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys
import unittest

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import re
import tempfile

from yt_dlp.utils import YoutubeDLCookieJar

class TestYoutubeDLCookieJar(unittest.TestCase):
    def test_keep_session_cookies(self):
        cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt')
        cookiejar.load(ignore_discard=True, ignore_expires=True)
        tf = tempfile.NamedTemporaryFile(delete=False)
        try:
            cookiejar.save(filename=tf.name, ignore_discard=True, ignore_expires=True)
            temp = tf.read().decode()
            self.assertTrue(re.search(
                r'www\.foobar\.foobar\s+FALSE\s+/\s+TRUE\s+0\s+YoutubeDLExpiresEmpty\s+YoutubeDLExpiresEmptyValue', temp))
            self.assertTrue(re.search(
                r'www\.foobar\.foobar\s+FALSE\s+/\s+TRUE\s+0\s+YoutubeDLExpires0\s+YoutubeDLExpires0Value', temp))
        finally:
            tf.close()
            os.remove(tf.name)

    def test_strip_httponly_prefix(self):
        cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt')
        cookiejar.load(ignore_discard=True, ignore_expires=True)

        def assert_cookie_has_value(key):
            self.assertEqual(cookiejar._cookies['www.foobar.foobar']['/'][key].value, key + '_VALUE')

        assert_cookie_has_value('HTTPONLY_COOKIE')
        assert_cookie_has_value('JS_ACCESSIBLE_COOKIE')

    def test_malformed_cookies(self):
        cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/malformed_cookies.txt')
        cookiejar.load(ignore_discard=True, ignore_expires=True)
        # Cookies should be empty since all malformed cookie file entries
        # will be ignored
        self.assertFalse(cookiejar._cookies)

if __name__ == '__main__':
    unittest.main()

repo_name: Lesmiscore/ytdl


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys
import unittest

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import base64

from yt_dlp.aes import (
    BLOCK_SIZE_BYTES,
    aes_cbc_decrypt,
    aes_cbc_decrypt_bytes,
    aes_cbc_encrypt,
    aes_ctr_decrypt,
    aes_ctr_encrypt,
    aes_decrypt,
    aes_decrypt_text,
    aes_ecb_decrypt,
    aes_ecb_encrypt,
    aes_encrypt,
    aes_gcm_decrypt_and_verify,
    aes_gcm_decrypt_and_verify_bytes,
    key_expansion,
    pad_block,
)
from yt_dlp.dependencies import Cryptodome_AES
from yt_dlp.utils import bytes_to_intlist, intlist_to_bytes

# the encrypted data can be generate with 'devscripts/generate_aes_testdata.py'

class TestAES(unittest.TestCase):
    def setUp(self):
        self.key = self.iv = [0x20, 0x15] + 14 * [0]
        self.secret_msg = b'Secret message goes here'

    def test_encrypt(self):
        msg = b'message'
        key = list(range(16))
        encrypted = aes_encrypt(bytes_to_intlist(msg), key)
        decrypted = intlist_to_bytes(aes_decrypt(encrypted, key))
        self.assertEqual(decrypted, msg)

    def test_cbc_decrypt(self):
        data = b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\x27\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd'
        decrypted = intlist_to_bytes(aes_cbc_decrypt(bytes_to_intlist(data), self.key, self.iv))
        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
        if Cryptodome_AES:
            decrypted = aes_cbc_decrypt_bytes(data, intlist_to_bytes(self.key), intlist_to_bytes(self.iv))
            self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)

    def test_cbc_encrypt(self):
        data = bytes_to_intlist(self.secret_msg)
        encrypted = intlist_to_bytes(aes_cbc_encrypt(data, self.key, self.iv))
        self.assertEqual(
            encrypted,
            b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd')

    def test_ctr_decrypt(self):
        data = bytes_to_intlist(b'\x03\xc7\xdd\xd4\x8e\xb3\xbc\x1a*O\xdc1\x12+8Aio\xd1z\xb5#\xaf\x08')
        decrypted = intlist_to_bytes(aes_ctr_decrypt(data, self.key, self.iv))
        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)

    def test_ctr_encrypt(self):
        data = bytes_to_intlist(self.secret_msg)
        encrypted = intlist_to_bytes(aes_ctr_encrypt(data, self.key, self.iv))
        self.assertEqual(
            encrypted,
            b'\x03\xc7\xdd\xd4\x8e\xb3\xbc\x1a*O\xdc1\x12+8Aio\xd1z\xb5#\xaf\x08')

    def test_gcm_decrypt(self):
        data = b'\x159Y\xcf5eud\x90\x9c\x85&]\x14\x1d\x0f.\x08\xb4T\xe4/\x17\xbd'
        authentication_tag = b'\xe8&I\x80rI\x07\x9d}YWuU@:e'

        decrypted = intlist_to_bytes(aes_gcm_decrypt_and_verify(
            bytes_to_intlist(data), self.key, bytes_to_intlist(authentication_tag), self.iv[:12]))
        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
        if Cryptodome_AES:
            decrypted = aes_gcm_decrypt_and_verify_bytes(
                data, intlist_to_bytes(self.key), authentication_tag, intlist_to_bytes(self.iv[:12]))
            self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)

    def test_decrypt_text(self):
        password = intlist_to_bytes(self.key).decode()
        encrypted = base64.b64encode(
            intlist_to_bytes(self.iv[:8])
            + b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae'
        ).decode()
        decrypted = (aes_decrypt_text(encrypted, password, 16))
        self.assertEqual(decrypted, self.secret_msg)

        password = intlist_to_bytes(self.key).decode()
        encrypted = base64.b64encode(
            intlist_to_bytes(self.iv[:8])
            + b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83'
        ).decode()
        decrypted = (aes_decrypt_text(encrypted, password, 32))
        self.assertEqual(decrypted, self.secret_msg)

    def test_ecb_encrypt(self):
        data = bytes_to_intlist(self.secret_msg)
        data += [0x08] * (BLOCK_SIZE_BYTES - len(data) % BLOCK_SIZE_BYTES)
        encrypted = intlist_to_bytes(aes_ecb_encrypt(data, self.key, self.iv))
        self.assertEqual(
            encrypted,
            b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:')

    def test_ecb_decrypt(self):
        data = bytes_to_intlist(b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:')
        decrypted = intlist_to_bytes(aes_ecb_decrypt(data, self.key, self.iv))
        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)

    def test_key_expansion(self):
        key = '4f6bdaa39e2f8cb07f5e722d9edef314'

        self.assertEqual(key_expansion(bytes_to_intlist(bytearray.fromhex(key))), [
            0x4F, 0x6B, 0xDA, 0xA3, 0x9E, 0x2F, 0x8C, 0xB0, 0x7F, 0x5E, 0x72, 0x2D, 0x9E, 0xDE, 0xF3, 0x14,
            0x53, 0x66, 0x20, 0xA8, 0xCD, 0x49, 0xAC, 0x18, 0xB2, 0x17, 0xDE, 0x35, 0x2C, 0xC9, 0x2D, 0x21,
            0x8C, 0xBE, 0xDD, 0xD9, 0x41, 0xF7, 0x71, 0xC1, 0xF3, 0xE0, 0xAF, 0xF4, 0xDF, 0x29, 0x82, 0xD5,
            0x2D, 0xAD, 0xDE, 0x47, 0x6C, 0x5A, 0xAF, 0x86, 0x9F, 0xBA, 0x00, 0x72, 0x40, 0x93, 0x82, 0xA7,
            0xF9, 0xBE, 0x82, 0x4E, 0x95, 0xE4, 0x2D, 0xC8, 0x0A, 0x5E, 0x2D, 0xBA, 0x4A, 0xCD, 0xAF, 0x1D,
            0x54, 0xC7, 0x26, 0x98, 0xC1, 0x23, 0x0B, 0x50, 0xCB, 0x7D, 0x26, 0xEA, 0x81, 0xB0, 0x89, 0xF7,
            0x93, 0x60, 0x4E, 0x94, 0x52, 0x43, 0x45, 0xC4, 0x99, 0x3E, 0x63, 0x2E, 0x18, 0x8E, 0xEA, 0xD9,
            0xCA, 0xE7, 0x7B, 0x39, 0x98, 0xA4, 0x3E, 0xFD, 0x01, 0x9A, 0x5D, 0xD3, 0x19, 0x14, 0xB7, 0x0A,
            0xB0, 0x4E, 0x1C, 0xED, 0x28, 0xEA, 0x22, 0x10, 0x29, 0x70, 0x7F, 0xC3, 0x30, 0x64, 0xC8, 0xC9,
            0xE8, 0xA6, 0xC1, 0xE9, 0xC0, 0x4C, 0xE3, 0xF9, 0xE9, 0x3C, 0x9C, 0x3A, 0xD9, 0x58, 0x54, 0xF3,
            0xB4, 0x86, 0xCC, 0xDC, 0x74, 0xCA, 0x2F, 0x25, 0x9D, 0xF6, 0xB3, 0x1F, 0x44, 0xAE, 0xE7, 0xEC])

    def test_pad_block(self):
        block = [0x21, 0xA0, 0x43, 0xFF]

        self.assertEqual(pad_block(block, 'pkcs7'),
                         block + [0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C])

        self.assertEqual(pad_block(block, 'iso7816'),
                         block + [0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])

        self.assertEqual(pad_block(block, 'whitespace'),
                         block + [0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20])

        self.assertEqual(pad_block(block, 'zero'),
                         block + [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])

        block = list(range(16))
        for mode in ('pkcs7', 'iso7816', 'whitespace', 'zero'):
            self.assertEqual(pad_block(block, mode), block, mode)

if __name__ == '__main__':
    unittest.main()


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys
import unittest

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from test.helper import is_download_test, try_rm
from yt_dlp import YoutubeDL

def _download_restricted(url, filename, age):
    """ Returns true if the file has been downloaded """

    params = {
        'age_limit': age,
        'skip_download': True,
        'writeinfojson': True,
        'outtmpl': '%(id)s.%(ext)s',
    }
    ydl = YoutubeDL(params)
    ydl.add_default_info_extractors()
    json_filename = os.path.splitext(filename)[0] + '.info.json'
    try_rm(json_filename)
    ydl.download([url])
    res = os.path.exists(json_filename)
    try_rm(json_filename)
    return res

@is_download_test
class TestAgeRestriction(unittest.TestCase):
    def _assert_restricted(self, url, filename, age, old_age=None):
        self.assertTrue(_download_restricted(url, filename, old_age))
        self.assertFalse(_download_restricted(url, filename, age))

    def test_youtube(self):
        self._assert_restricted('07FYdnEawAQ', '07FYdnEawAQ.mp4', 10)

    def test_youporn(self):
        self._assert_restricted(
            'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
            '505835.mp4', 2, old_age=25)

if __name__ == '__main__':
    unittest.main()


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys
import unittest

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import collections

from test.helper import gettestcases
from yt_dlp.extractor import FacebookIE, YoutubeIE, gen_extractors

class TestAllURLsMatching(unittest.TestCase):
    def setUp(self):
        self.ies = gen_extractors()

    def matching_ies(self, url):
        return [ie.IE_NAME for ie in self.ies if ie.suitable(url) and ie.IE_NAME not in ('generic', 'Streamlink')]

    def assertMatch(self, url, ie_list):
        self.assertEqual(self.matching_ies(url), ie_list)

    def test_youtube_playlist_matching(self):
        assertPlaylist = lambda url: self.assertMatch(url, ['youtube:playlist'])
        assertTab = lambda url: self.assertMatch(url, ['youtube:tab'])
        assertPlaylist('ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
        assertPlaylist('UUBABnxM4Ar9ten8Mdjj1j0Q')  # 585
        assertPlaylist('PL63F0C78739B09958')
        assertTab('https://www.youtube.com/AsapSCIENCE')
        assertTab('https://www.youtube.com/embedded')
        assertTab('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
        assertTab('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')
        assertTab('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')  # 668
        self.assertFalse('youtube:playlist' in self.matching_ies('PLtS2H6bU1M'))
        # Top tracks
        assertTab('https://www.youtube.com/playlist?list=MCUS.20142101')

    def test_youtube_matching(self):
        self.assertTrue(YoutubeIE.suitable('PLtS2H6bU1M'))
        self.assertFalse(YoutubeIE.suitable('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012'))  # 668
        self.assertMatch('http://youtu.be/BaW_jenozKc', ['youtube'])
        # self.assertMatch('http://www.youtube.com/v/BaW_jenozKc', ['youtube'])  # /v/ is no longer valid
        self.assertMatch('https://youtube.googleapis.com/v/BaW_jenozKc', ['youtube'])
        self.assertMatch('http://www.cleanvideosearch.com/media/action/yt/watch?videoId=8v_4O44sfjM', ['youtube'])

    def test_youtube_channel_matching(self):
        assertChannel = lambda url: self.assertMatch(url, ['youtube:tab'])
        assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM')
        assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM?feature=gb_ch_rec')
        assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')

    def test_youtube_user_matching(self):
        self.assertMatch('http://www.youtube.com/NASAgovVideo/videos', ['youtube:tab'])

    def test_youtube_feeds(self):
        self.assertMatch('https://www.youtube.com/feed/library', ['youtube:tab'])
        self.assertMatch('https://www.youtube.com/feed/history', ['youtube:tab'])
        self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:tab'])
        self.assertMatch('https://www.youtube.com/feed/subscriptions', ['youtube:tab'])

    def test_youtube_search_matching(self):
        self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
        self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])

    def test_facebook_matching(self):
        self.assertTrue(FacebookIE.suitable('https://www.facebook.com/Shiniknoh#!/photo.php?v=10153317450565268'))
        self.assertTrue(FacebookIE.suitable('https://www.facebook.com/cindyweather?fref=ts#!/photo.php?v=10152183998945793'))

    def test_no_duplicates(self):
        ies = gen_extractors()
        for tc in gettestcases(include_onlymatching=True):
            url = tc['url']
            for ie in ies:
                if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'):
                    self.assertTrue(ie.suitable(url), f'{type(ie).__name__} should match URL {url!r}')
                else:
                    self.assertFalse(
                        ie.suitable(url),
                        f'{type(ie).__name__} should not match URL {url!r} . That URL belongs to {tc["name"]}.')

    def test_keywords(self):
        self.assertMatch(':ytsubs', ['youtube:subscriptions'])
        self.assertMatch(':ytsubscriptions', ['youtube:subscriptions'])
        self.assertMatch(':ythistory', ['youtube:history'])

    def test_vimeo_matching(self):
        self.assertMatch('https://vimeo.com/channels/tributes', ['vimeo:channel'])
        self.assertMatch('https://vimeo.com/channels/31259', ['vimeo:channel'])
        self.assertMatch('https://vimeo.com/channels/31259/53576664', ['vimeo'])
        self.assertMatch('https://vimeo.com/user7108434', ['vimeo:user'])
        self.assertMatch('https://vimeo.com/user7108434/videos', ['vimeo:user'])
        self.assertMatch('https://vimeo.com/user21297594/review/75524534/3c257a1b5d', ['vimeo:review'])

    # https://github.com/ytdl-org/youtube-dl/issues/1930
    def test_soundcloud_not_matching_sets(self):
        self.assertMatch('http://soundcloud.com/floex/sets/gone-ep', ['soundcloud:set'])

    def test_tumblr(self):
        self.assertMatch('http://tatianamaslanydaily.tumblr.com/post/54196191430/orphan-black-dvd-extra-behind-the-scenes', ['Tumblr'])
        self.assertMatch('http://tatianamaslanydaily.tumblr.com/post/54196191430', ['Tumblr'])

    def test_pbs(self):
        # https://github.com/ytdl-org/youtube-dl/issues/2350
        self.assertMatch('http://video.pbs.org/viralplayer/2365173446/', ['pbs'])
        self.assertMatch('http://video.pbs.org/widget/partnerplayer/980042464/', ['pbs'])

    def test_no_duplicated_ie_names(self):
        name_accu = collections.defaultdict(list)
        for ie in self.ies:
            name_accu[ie.IE_NAME.lower()].append(type(ie).__name__)
        for (ie_name, ie_list) in name_accu.items():
            self.assertEqual(
                len(ie_list), 1,
                f'Multiple extractors with the same IE_NAME "{ie_name}" ({", ".join(ie_list)})')

if __name__ == '__main__':
    unittest.main()


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys
import unittest

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import shutil

from test.helper import FakeYDL
from yt_dlp.cache import Cache

def _is_empty(d):
    return not bool(os.listdir(d))

def _mkdir(d):
    if not os.path.exists(d):
        os.mkdir(d)

class TestCache(unittest.TestCase):
    def setUp(self):
        TEST_DIR = os.path.dirname(os.path.abspath(__file__))
        TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata')
        _mkdir(TESTDATA_DIR)
        self.test_dir = os.path.join(TESTDATA_DIR, 'cache_test')
        self.tearDown()

    def tearDown(self):
        if os.path.exists(self.test_dir):
            shutil.rmtree(self.test_dir)

    def test_cache(self):
        ydl = FakeYDL({
            'cachedir': self.test_dir,
        })
        c = Cache(ydl)
        obj = {'x': 1, 'y': ['ä', '\\a', True]}
        self.assertEqual(c.load('test_cache', 'k.'), None)
        c.store('test_cache', 'k.', obj)
        self.assertEqual(c.load('test_cache', 'k2'), None)
        self.assertFalse(_is_empty(self.test_dir))
        self.assertEqual(c.load('test_cache', 'k.'), obj)
        self.assertEqual(c.load('test_cache', 'y'), None)
        self.assertEqual(c.load('test_cache2', 'k.'), None)
        c.remove()
        self.assertFalse(os.path.exists(self.test_dir))
        self.assertEqual(c.load('test_cache', 'k.'), None)

if __name__ == '__main__':
    unittest.main()


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys
import unittest

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import struct
import urllib.parse

from yt_dlp import compat
from yt_dlp.compat import (
    compat_etree_fromstring,
    compat_expanduser,
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlencode,
)

class TestCompat(unittest.TestCase):
    def test_compat_passthrough(self):
        with self.assertWarns(DeprecationWarning):
            compat.compat_basestring

        with self.assertWarns(DeprecationWarning):
            compat.WINDOWS_VT_MODE

        # TODO: Test submodule
        # compat.asyncio.events  # Must not raise error

    def test_compat_expanduser(self):
        old_home = os.environ.get('HOME')
        test_str = R'C:\Documents and Settings\тест\Application Data'
        try:
            os.environ['HOME'] = test_str
            self.assertEqual(compat_expanduser('~'), test_str)
        finally:
            os.environ['HOME'] = old_home or ''

    def test_compat_urllib_parse_unquote(self):
        self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def')
        self.assertEqual(compat_urllib_parse_unquote('%7e/abc+def'), '~/abc+def')
        self.assertEqual(compat_urllib_parse_unquote(''), '')
        self.assertEqual(compat_urllib_parse_unquote('%'), '%')
        self.assertEqual(compat_urllib_parse_unquote('%%'), '%%')
        self.assertEqual(compat_urllib_parse_unquote('%%%'), '%%%')
        self.assertEqual(compat_urllib_parse_unquote('%2F'), '/')
        self.assertEqual(compat_urllib_parse_unquote('%2f'), '/')
        self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波')
        self.assertEqual(
            compat_urllib_parse_unquote('''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
%%a'''),
            '''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
%%a''')
        self.assertEqual(
            compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%'''),
            '''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''')

    def test_compat_urllib_parse_unquote_plus(self):
        self.assertEqual(urllib.parse.unquote_plus('abc%20def'), 'abc def')
        self.assertEqual(urllib.parse.unquote_plus('%7e/abc+def'), '~/abc def')

    def test_compat_urllib_parse_urlencode(self):
        self.assertEqual(compat_urllib_parse_urlencode({'abc': 'def'}), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode({'abc': b'def'}), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode({b'abc': 'def'}), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode({b'abc': b'def'}), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode([('abc', 'def')]), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode([('abc', b'def')]), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode([(b'abc', 'def')]), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode([(b'abc', b'def')]), 'abc=def')

    def test_compat_etree_fromstring(self):
        xml = '''
            <root foo="bar" spam="中文">
                <normal>foo</normal>
                <chinese>中文</chinese>
                <foo><bar>spam</bar></foo>
            </root>
        '''
        doc = compat_etree_fromstring(xml.encode())
        self.assertTrue(isinstance(doc.attrib['foo'], str))
        self.assertTrue(isinstance(doc.attrib['spam'], str))
        self.assertTrue(isinstance(doc.find('normal').text, str))
        self.assertTrue(isinstance(doc.find('chinese').text, str))
        self.assertTrue(isinstance(doc.find('foo/bar').text, str))

    def test_compat_etree_fromstring_doctype(self):
        xml = '''<?xml version="1.0"?>
<!DOCTYPE smil PUBLIC "-//W3C//DTD SMIL 2.0//EN" "http://www.w3.org/2001/SMIL20/SMIL20.dtd">
<smil xmlns="http://www.w3.org/2001/SMIL20/Language"></smil>'''
        compat_etree_fromstring(xml)

    def test_struct_unpack(self):
        self.assertEqual(struct.unpack('!B', b'\x00'), (0,))

if __name__ == '__main__':
    unittest.main()


code:
import unittest
from datetime import datetime, timezone

from yt_dlp import cookies
from yt_dlp.cookies import (
    LenientSimpleCookie,
    LinuxChromeCookieDecryptor,
    MacChromeCookieDecryptor,
    WindowsChromeCookieDecryptor,
    _get_linux_desktop_environment,
    _LinuxDesktopEnvironment,
    parse_safari_cookies,
    pbkdf2_sha1,
)

class Logger:
    def debug(self, message, *args, **kwargs):
        print(f'[verbose] {message}')

    def info(self, message, *args, **kwargs):
        print(message)

    def warning(self, message, *args, **kwargs):
        self.error(message)

    def error(self, message, *args, **kwargs):
        raise Exception(message)

class MonkeyPatch:
    def __init__(self, module, temporary_values):
        self._module = module
        self._temporary_values = temporary_values
        self._backup_values = {}

    def __enter__(self):
        for name, temp_value in self._temporary_values.items():
            self._backup_values[name] = getattr(self._module, name)
            setattr(self._module, name, temp_value)

    def __exit__(self, exc_type, exc_val, exc_tb):
        for name, backup_value in self._backup_values.items():
            setattr(self._module, name, backup_value)

class TestCookies(unittest.TestCase):
    def test_get_desktop_environment(self):
        """ based on https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util_unittest.cc """
        test_cases = [
            ({}, _LinuxDesktopEnvironment.OTHER),

            ({'DESKTOP_SESSION': 'gnome'}, _LinuxDesktopEnvironment.GNOME),
            ({'DESKTOP_SESSION': 'mate'}, _LinuxDesktopEnvironment.GNOME),
            ({'DESKTOP_SESSION': 'kde4'}, _LinuxDesktopEnvironment.KDE),
            ({'DESKTOP_SESSION': 'kde'}, _LinuxDesktopEnvironment.KDE),
            ({'DESKTOP_SESSION': 'xfce'}, _LinuxDesktopEnvironment.XFCE),

            ({'GNOME_DESKTOP_SESSION_ID': 1}, _LinuxDesktopEnvironment.GNOME),
            ({'KDE_FULL_SESSION': 1}, _LinuxDesktopEnvironment.KDE),

            ({'XDG_CURRENT_DESKTOP': 'X-Cinnamon'}, _LinuxDesktopEnvironment.CINNAMON),
            ({'XDG_CURRENT_DESKTOP': 'GNOME'}, _LinuxDesktopEnvironment.GNOME),
            ({'XDG_CURRENT_DESKTOP': 'GNOME:GNOME-Classic'}, _LinuxDesktopEnvironment.GNOME),
            ({'XDG_CURRENT_DESKTOP': 'GNOME : GNOME-Classic'}, _LinuxDesktopEnvironment.GNOME),

            ({'XDG_CURRENT_DESKTOP': 'Unity', 'DESKTOP_SESSION': 'gnome-fallback'}, _LinuxDesktopEnvironment.GNOME),
            ({'XDG_CURRENT_DESKTOP': 'KDE', 'KDE_SESSION_VERSION': '5'}, _LinuxDesktopEnvironment.KDE),
            ({'XDG_CURRENT_DESKTOP': 'KDE'}, _LinuxDesktopEnvironment.KDE),
            ({'XDG_CURRENT_DESKTOP': 'Pantheon'}, _LinuxDesktopEnvironment.PANTHEON),
            ({'XDG_CURRENT_DESKTOP': 'Unity'}, _LinuxDesktopEnvironment.UNITY),
            ({'XDG_CURRENT_DESKTOP': 'Unity:Unity7'}, _LinuxDesktopEnvironment.UNITY),
            ({'XDG_CURRENT_DESKTOP': 'Unity:Unity8'}, _LinuxDesktopEnvironment.UNITY),
        ]

        for env, expected_desktop_environment in test_cases:
            self.assertEqual(_get_linux_desktop_environment(env), expected_desktop_environment)

    def test_chrome_cookie_decryptor_linux_derive_key(self):
        key = LinuxChromeCookieDecryptor.derive_key(b'abc')
        self.assertEqual(key, b'7\xa1\xec\xd4m\xfcA\xc7\xb19Z\xd0\x19\xdcM\x17')

    def test_chrome_cookie_decryptor_mac_derive_key(self):
        key = MacChromeCookieDecryptor.derive_key(b'abc')
        self.assertEqual(key, b'Y\xe2\xc0\xd0P\xf6\xf4\xe1l\xc1\x8cQ\xcb|\xcdY')

    def test_chrome_cookie_decryptor_linux_v10(self):
        with MonkeyPatch(cookies, {'_get_linux_keyring_password': lambda *args, **kwargs: b''}):
            encrypted_value = b'v10\xccW%\xcd\xe6\xe6\x9fM" \xa7\xb0\xca\xe4\x07\xd6'
            value = 'USD'
            decryptor = LinuxChromeCookieDecryptor('Chrome', Logger())
            self.assertEqual(decryptor.decrypt(encrypted_value), value)

    def test_chrome_cookie_decryptor_linux_v11(self):
        with MonkeyPatch(cookies, {'_get_linux_keyring_password': lambda *args, **kwargs: b''}):
            encrypted_value = b'v11#\x81\x10>`w\x8f)\xc0\xb2\xc1\r\xf4\x1al\xdd\x93\xfd\xf8\xf8N\xf2\xa9\x83\xf1\xe9o\x0elVQd'
            value = 'tz=Europe.London'
            decryptor = LinuxChromeCookieDecryptor('Chrome', Logger())
            self.assertEqual(decryptor.decrypt(encrypted_value), value)

    def test_chrome_cookie_decryptor_windows_v10(self):
        with MonkeyPatch(cookies, {
            '_get_windows_v10_key': lambda *args, **kwargs: b'Y\xef\xad\xad\xeerp\xf0Y\xe6\x9b\x12\xc2<z\x16]\n\xbb\xb8\xcb\xd7\x9bA\xc3\x14e\x99{\xd6\xf4&'
        }):
            encrypted_value = b'v10T\xb8\xf3\xb8\x01\xa7TtcV\xfc\x88\xb8\xb8\xef\x05\xb5\xfd\x18\xc90\x009\xab\xb1\x893\x85)\x87\xe1\xa9-\xa3\xad='
            value = '32101439'
            decryptor = WindowsChromeCookieDecryptor('', Logger())
            self.assertEqual(decryptor.decrypt(encrypted_value), value)

    def test_chrome_cookie_decryptor_mac_v10(self):
        with MonkeyPatch(cookies, {'_get_mac_keyring_password': lambda *args, **kwargs: b'6eIDUdtKAacvlHwBVwvg/Q=='}):
            encrypted_value = b'v10\xb3\xbe\xad\xa1[\x9fC\xa1\x98\xe0\x9a\x01\xd9\xcf\xbfc'
            value = '2021-06-01-22'
            decryptor = MacChromeCookieDecryptor('', Logger())
            self.assertEqual(decryptor.decrypt(encrypted_value), value)

    def test_safari_cookie_parsing(self):
        cookies = \
            b'cook\x00\x00\x00\x01\x00\x00\x00i\x00\x00\x01\x00\x01\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00Y' \
            b'\x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x008\x00\x00\x00B\x00\x00\x00F\x00\x00\x00H' \
            b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x03\xa5>\xc3A\x00\x00\x80\xc3\x07:\xc3A' \
            b'localhost\x00foo\x00/\x00test%20%3Bcookie\x00\x00\x00\x054\x07\x17 \x05\x00\x00\x00Kbplist00\xd1\x01' \
            b'\x02_\x10\x18NSHTTPCookieAcceptPolicy\x10\x02\x08\x0b&\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00' \
            b'\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00('

        jar = parse_safari_cookies(cookies)
        self.assertEqual(len(jar), 1)
        cookie = list(jar)[0]
        self.assertEqual(cookie.domain, 'localhost')
        self.assertEqual(cookie.port, None)
        self.assertEqual(cookie.path, '/')
        self.assertEqual(cookie.name, 'foo')
        self.assertEqual(cookie.value, 'test%20%3Bcookie')
        self.assertFalse(cookie.secure)
        expected_expiration = datetime(2021, 6, 18, 21, 39, 19, tzinfo=timezone.utc)
        self.assertEqual(cookie.expires, int(expected_expiration.timestamp()))

    def test_pbkdf2_sha1(self):
        key = pbkdf2_sha1(b'peanuts', b' ' * 16, 1, 16)
        self.assertEqual(key, b'g\xe1\x8e\x0fQ\x1c\x9b\xf3\xc9`!\xaa\x90\xd9\xd34')

class TestLenientSimpleCookie(unittest.TestCase):
    def _run_tests(self, *cases):
        for message, raw_cookie, expected in cases:
            cookie = LenientSimpleCookie(raw_cookie)

            with self.subTest(message, expected=expected):
                self.assertEqual(cookie.keys(), expected.keys(), message)

                for key, expected_value in expected.items():
                    morsel = cookie[key]
                    if isinstance(expected_value, tuple):
                        expected_value, expected_attributes = expected_value
                    else:
                        expected_attributes = {}

                    attributes = {
                        key: value
                        for key, value in dict(morsel).items()
                        if value != ""
                    }
                    self.assertEqual(attributes, expected_attributes, message)

                    self.assertEqual(morsel.value, expected_value, message)

    def test_parsing(self):
        self._run_tests(
            # Copied from https://github.com/python/cpython/blob/v3.10.7/Lib/test/test_http_cookies.py
            (
                "Test basic cookie",
                "chips=ahoy; vienna=finger",
                {"chips": "ahoy", "vienna": "finger"},
            ),
            (
                "Test quoted cookie",
                'keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"',
                {"keebler": 'E=mc2; L="Loves"; fudge=\012;'},
            ),
            (
                "Allow '=' in an unquoted value",
                "keebler=E=mc2",
                {"keebler": "E=mc2"},
            ),
            (
                "Allow cookies with ':' in their name",
                "key:term=value:term",
                {"key:term": "value:term"},
            ),
            (
                "Allow '[' and ']' in cookie values",
                "a=b; c=[; d=r; f=h",
                {"a": "b", "c": "[", "d": "r", "f": "h"},
            ),
            (
                "Test basic cookie attributes",
                'Customer="WILE_E_COYOTE"; Version=1; Path=/acme',
                {"Customer": ("WILE_E_COYOTE", {"version": "1", "path": "/acme"})},
            ),
            (
                "Test flag only cookie attributes",
                'Customer="WILE_E_COYOTE"; HttpOnly; Secure',
                {"Customer": ("WILE_E_COYOTE", {"httponly": True, "secure": True})},
            ),
            (
                "Test flag only attribute with values",
                "eggs=scrambled; httponly=foo; secure=bar; Path=/bacon",
                {"eggs": ("scrambled", {"httponly": "foo", "secure": "bar", "path": "/bacon"})},
            ),
            (
                "Test special case for 'expires' attribute, 4 digit year",
                'Customer="W"; expires=Wed, 01 Jan 2010 00:00:00 GMT',
                {"Customer": ("W", {"expires": "Wed, 01 Jan 2010 00:00:00 GMT"})},
            ),
            (
                "Test special case for 'expires' attribute, 2 digit year",
                'Customer="W"; expires=Wed, 01 Jan 98 00:00:00 GMT',
                {"Customer": ("W", {"expires": "Wed, 01 Jan 98 00:00:00 GMT"})},
            ),
            (
                "Test extra spaces in keys and values",
                "eggs  =  scrambled  ;  secure  ;  path  =  bar   ; foo=foo   ",
                {"eggs": ("scrambled", {"secure": True, "path": "bar"}), "foo": "foo"},
            ),
            (
                "Test quoted attributes",
                'Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"',
                {"Customer": ("WILE_E_COYOTE", {"version": "1", "path": "/acme"})}
            ),
            # Our own tests that CPython passes
            (
                "Allow ';' in quoted value",
                'chips="a;hoy"; vienna=finger',
                {"chips": "a;hoy", "vienna": "finger"},
            ),
            (
                "Keep only the last set value",
                "a=c; a=b",
                {"a": "b"},
            ),
        )

    def test_lenient_parsing(self):
        self._run_tests(
            (
                "Ignore and try to skip invalid cookies",
                'chips={"ahoy;": 1}; vienna="finger;"',
                {"vienna": "finger;"},
            ),
            (
                "Ignore cookies without a name",
                "a=b; unnamed; c=d",
                {"a": "b", "c": "d"},
            ),
            (
                "Ignore '\"' cookie without name",
                'a=b; "; c=d',
                {"a": "b", "c": "d"},
            ),
            (
                "Skip all space separated values",
                "x a=b c=d x; e=f",
                {"a": "b", "c": "d", "e": "f"},
            ),
            (
                "Skip all space separated values",
                'x a=b; data={"complex": "json", "with": "key=value"}; x c=d x',
                {"a": "b", "c": "d"},
            ),
            (
                "Expect quote mending",
                'a=b; invalid="; c=d',
                {"a": "b", "c": "d"},
            ),
            (
                "Reset morsel after invalid to not capture attributes",
                "a=b; invalid; Version=1; c=d",
                {"a": "b", "c": "d"},
            ),
            (
                "Continue after non-flag attribute without value",
                "a=b; path; Version=1; c=d",
                {"a": "b", "c": "d"},
            ),
        )


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys
import unittest

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import collections
import hashlib
import http.client
import json
import socket
import urllib.error

from test.helper import (
    assertGreaterEqual,
    expect_info_dict,
    expect_warnings,
    get_params,
    gettestcases,
    getwebpagetestcases,
    is_download_test,
    report_warning,
    try_rm,
)

import yt_dlp.YoutubeDL  # isort: split
from yt_dlp.extractor import get_info_extractor
from yt_dlp.utils import (
    DownloadError,
    ExtractorError,
    UnavailableVideoError,
    format_bytes,
    join_nonempty,
)

RETRIES = 3

class YoutubeDL(yt_dlp.YoutubeDL):
    def __init__(self, *args, **kwargs):
        self.to_stderr = self.to_screen
        self.processed_info_dicts = []
        super().__init__(*args, **kwargs)

    def report_warning(self, message, *args, **kwargs):
        # Don't accept warnings during tests
        raise ExtractorError(message)

    def process_info(self, info_dict):
        self.processed_info_dicts.append(info_dict.copy())
        return super().process_info(info_dict)

def _file_md5(fn):
    with open(fn, 'rb') as f:
        return hashlib.md5(f.read()).hexdigest()

normal_test_cases = gettestcases()
webpage_test_cases = getwebpagetestcases()
tests_counter = collections.defaultdict(collections.Counter)

@is_download_test
class TestDownload(unittest.TestCase):
    # Parallel testing in nosetests. See
    # http://nose.readthedocs.org/en/latest/doc_tests/test_multiprocess/multiprocess.html
    _multiprocess_shared_ = True

    maxDiff = None

    COMPLETED_TESTS = {}

    def __str__(self):
        """Identify each test with the `add_ie` attribute, if available."""
        cls, add_ie = type(self), getattr(self, self._testMethodName).add_ie
        return f'{self._testMethodName} ({cls.__module__}.{cls.__name__}){f" [{add_ie}]" if add_ie else ""}:'

# Dynamically generate tests

def generator(test_case, tname):
    def test_template(self):
        if self.COMPLETED_TESTS.get(tname):
            return
        self.COMPLETED_TESTS[tname] = True
        ie = yt_dlp.extractor.get_info_extractor(test_case['name'])()
        other_ies = [get_info_extractor(ie_key)() for ie_key in test_case.get('add_ie', [])]
        is_playlist = any(k.startswith('playlist') for k in test_case)
        test_cases = test_case.get(
            'playlist', [] if is_playlist else [test_case])

        def print_skipping(reason):
            print('Skipping %s: %s' % (test_case['name'], reason))
            self.skipTest(reason)

        if not ie.working():
            print_skipping('IE marked as not _WORKING')

        for tc in test_cases:
            info_dict = tc.get('info_dict', {})
            params = tc.get('params', {})
            if not info_dict.get('id'):
                raise Exception(f'Test {tname} definition incorrect - "id" key is not present')
            elif not info_dict.get('ext'):
                if params.get('skip_download') and params.get('ignore_no_formats_error'):
                    continue
                raise Exception(f'Test {tname} definition incorrect - "ext" key must be present to define the output file')

        if 'skip' in test_case:
            print_skipping(test_case['skip'])

        for other_ie in other_ies:
            if not other_ie.working():
                print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key())

        params = get_params(test_case.get('params', {}))
        params['outtmpl'] = tname + '_' + params['outtmpl']
        if is_playlist and 'playlist' not in test_case:
            params.setdefault('extract_flat', 'in_playlist')
            params.setdefault('playlistend', test_case.get('playlist_mincount'))
            params.setdefault('skip_download', True)

        ydl = YoutubeDL(params, auto_init=False)
        ydl.add_default_info_extractors()
        finished_hook_called = set()

        def _hook(status):
            if status['status'] == 'finished':
                finished_hook_called.add(status['filename'])
        ydl.add_progress_hook(_hook)
        expect_warnings(ydl, test_case.get('expected_warnings', []))

        def get_tc_filename(tc):
            return ydl.prepare_filename(dict(tc.get('info_dict', {})))

        res_dict = None

        def try_rm_tcs_files(tcs=None):
            if tcs is None:
                tcs = test_cases
            for tc in tcs:
                tc_filename = get_tc_filename(tc)
                try_rm(tc_filename)
                try_rm(tc_filename + '.part')
                try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
        try_rm_tcs_files()
        try:
            try_num = 1
            while True:
                try:
                    # We're not using .download here since that is just a shim
                    # for outside error handling, and returns the exit code
                    # instead of the result dict.
                    res_dict = ydl.extract_info(
                        test_case['url'],
                        force_generic_extractor=params.get('force_generic_extractor', False))
                except (DownloadError, ExtractorError) as err:
                    # Check if the exception is not a network related one
                    if (err.exc_info[0] not in (urllib.error.URLError, socket.timeout, UnavailableVideoError, http.client.BadStatusLine)
                            or (err.exc_info[0] == urllib.error.HTTPError and err.exc_info[1].code == 503)):
                        err.msg = f'{getattr(err, "msg", err)} ({tname})'
                        raise

                    if try_num == RETRIES:
                        report_warning('%s failed due to network errors, skipping...' % tname)
                        return

                    print(f'Retrying: {try_num} failed tries\n\n##########\n\n')

                    try_num += 1
                else:
                    break

            if is_playlist:
                self.assertTrue(res_dict['_type'] in ['playlist', 'multi_video'])
                self.assertTrue('entries' in res_dict)
                expect_info_dict(self, res_dict, test_case.get('info_dict', {}))

            if 'playlist_mincount' in test_case:
                assertGreaterEqual(
                    self,
                    len(res_dict['entries']),
                    test_case['playlist_mincount'],
                    'Expected at least %d in playlist %s, but got only %d' % (
                        test_case['playlist_mincount'], test_case['url'],
                        len(res_dict['entries'])))
            if 'playlist_count' in test_case:
                self.assertEqual(
                    len(res_dict['entries']),
                    test_case['playlist_count'],
                    'Expected %d entries in playlist %s, but got %d.' % (
                        test_case['playlist_count'],
                        test_case['url'],
                        len(res_dict['entries']),
                    ))
            if 'playlist_duration_sum' in test_case:
                got_duration = sum(e['duration'] for e in res_dict['entries'])
                self.assertEqual(
                    test_case['playlist_duration_sum'], got_duration)

            # Generalize both playlists and single videos to unified format for
            # simplicity
            if 'entries' not in res_dict:
                res_dict['entries'] = [res_dict]

            for tc_num, tc in enumerate(test_cases):
                tc_res_dict = res_dict['entries'][tc_num]
                # First, check test cases' data against extracted data alone
                expect_info_dict(self, tc_res_dict, tc.get('info_dict', {}))
                # Now, check downloaded file consistency
                tc_filename = get_tc_filename(tc)
                if not test_case.get('params', {}).get('skip_download', False):
                    self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
                    self.assertTrue(tc_filename in finished_hook_called)
                    expected_minsize = tc.get('file_minsize', 10000)
                    if expected_minsize is not None:
                        if params.get('test'):
                            expected_minsize = max(expected_minsize, 10000)
                        got_fsize = os.path.getsize(tc_filename)
                        assertGreaterEqual(
                            self, got_fsize, expected_minsize,
                            'Expected %s to be at least %s, but it\'s only %s ' %
                            (tc_filename, format_bytes(expected_minsize),
                                format_bytes(got_fsize)))
                    if 'md5' in tc:
                        md5_for_file = _file_md5(tc_filename)
                        self.assertEqual(tc['md5'], md5_for_file)
                # Finally, check test cases' data again but this time against
                # extracted data from info JSON file written during processing
                info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
                self.assertTrue(
                    os.path.exists(info_json_fn),
                    'Missing info file %s' % info_json_fn)
                with open(info_json_fn, encoding='utf-8') as infof:
                    info_dict = json.load(infof)
                expect_info_dict(self, info_dict, tc.get('info_dict', {}))
        finally:
            try_rm_tcs_files()
            if is_playlist and res_dict is not None and res_dict.get('entries'):
                # Remove all other files that may have been extracted if the
                # extractor returns full results even with extract_flat
                res_tcs = [{'info_dict': e} for e in res_dict['entries']]
                try_rm_tcs_files(res_tcs)

    return test_template

# And add them to TestDownload
def inject_tests(test_cases, label=''):
    for test_case in test_cases:
        name = test_case['name']
        tname = join_nonempty('test', name, label, tests_counter[name][label], delim='_')
        tests_counter[name][label] += 1

        test_method = generator(test_case, tname)
        test_method.__name__ = tname
        test_method.add_ie = ','.join(test_case.get('add_ie', []))
        setattr(TestDownload, test_method.__name__, test_method)

inject_tests(normal_test_cases)

# TODO: disable redirection to the IE to ensure we are actually testing the webpage extraction
inject_tests(webpage_test_cases, 'webpage')

def batch_generator(name):
    def test_template(self):
        for label, num_tests in tests_counter[name].items():
            for i in range(num_tests):
                test_name = join_nonempty('test', name, label, i, delim='_')
                try:
                    getattr(self, test_name)()
                except unittest.SkipTest:
                    print(f'Skipped {test_name}')

    return test_template

for name in tests_counter:
    test_method = batch_generator(name)
    test_method.__name__ = f'test_{name}_all'
    test_method.add_ie = ''
    setattr(TestDownload, test_method.__name__, test_method)
del test_method

if __name__ == '__main__':
    unittest.main()