code:
#!/bin/sh
exec "${PYTHON:-python3}" -bb -Werror -Xdev "$(dirname "$(realpath "$0")")/yt_dlp/__main__.py" "$@"

repo_name: Lesmiscore/ytdl

path: b


code:
BSDIFF4023DBZh91AY&SY�g�mDD`@ !��`�Њe��H�
l�M�BZh91AY&SY>���  �Ml�
�%��rE8P�>�BZh9rE8P�


code:
BSDIFF4024DBZh91AY&SYk�.�DH`@ !��`�Њe��H�

te�`BZh91AY&SY֤�?�� 0�R�~���o��v�rE8P�֤�?BZh9rE8P�


code: # Empty file needed to make devscripts.utils properly importable from outside


code:
__youtube_dl()
{
    local cur prev opts fileopts diropts keywords
    COMPREPLY=()
    cur="${COMP_WORDS[COMP_CWORD]}"
    prev="${COMP_WORDS[COMP_CWORD-1]}"
    opts="{{flags}}"
    keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory"
    fileopts="-a|--batch-file|--download-archive|--cookies|--load-info"
    diropts="--cache-dir"

    if [[ ${prev} =~ ${fileopts} ]]; then
        COMPREPLY=( $(compgen -f -- ${cur}) )
        return 0
    elif [[ ${prev} =~ ${diropts} ]]; then
        COMPREPLY=( $(compgen -d -- ${cur}) )
        return 0
    fi

    if [[ ${cur} =~ : ]]; then
        COMPREPLY=( $(compgen -W "${keywords}" -- ${cur}) )
        return 0
    elif [[ ${cur} == * ]] ; then
        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
        return 0
    fi
}

complete -F __youtube_dl youtube-dl


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import yt_dlp

BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"

def build_completion(opt_parser):
    opts_flag = []
    for group in opt_parser.option_groups:
        for option in group.option_list:
            # for every long flag
            opts_flag.append(option.get_opt_string())
    with open(BASH_COMPLETION_TEMPLATE) as f:
        template = f.read()
    with open(BASH_COMPLETION_FILE, "w") as f:
        # just using the special char
        filled_template = template.replace("{{flags}}", " ".join(opts_flag))
        f.write(filled_template)

parser = yt_dlp.parseOpts(ignore_config_files=True)[0]
build_completion(parser)


code:
#!/usr/bin/env pwsh
# $IconName - red or white
# $BuilderName - PyInstaller or py2exe
Param($BuilderName, $IconName)
$ErrorActionPreference = "Stop"

switch ($BuilderName) {
    "PyInstaller" {
        $env:windows_icon="${IconName}"

        write-host "Building an EXE using PyInstaller"
        python -OO pyinst.py

        write-host "Moving built EXE into artifacts/"
        Move-Item dist/ytdl-patched.exe artifacts/
    }
    "py2exe" {
        $env:PY2EXE_WINDOWS_ICON_PATH="icons\youtube_social_squircle_${IconName}.ico"

        write-host "Building an EXE using py2exe"
        python setup.py py2exe

        write-host "Moving built EXE into artifacts/"
        Move-Item ytdl-patched.exe artifacts/
    }
    default {
        throw "Invalid BuilderName: $BuilderName"
    }
}


code:
#!/usr/bin/env python3
"""
This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check
if we are not 'age_limit' tagging some porn site

A second approach implemented relies on a list of porn domains, to activate it
pass the list filename as the only argument
"""

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import urllib.parse
import urllib.request

from test.helper import gettestcases

if len(sys.argv) > 1:
    METHOD = 'LIST'
    LIST = open(sys.argv[1]).read().decode('utf8').strip()
else:
    METHOD = 'EURISTIC'

for test in gettestcases():
    if METHOD == 'EURISTIC':
        try:
            webpage = urllib.request.urlopen(test['url'], timeout=10).read()
        except Exception:
            print('\nFail: {}'.format(test['name']))
            continue

        webpage = webpage.decode('utf8', 'replace')

        RESULT = 'porn' in webpage.lower()

    elif METHOD == 'LIST':
        domain = urllib.parse.urlparse(test['url']).netloc
        if not domain:
            print('\nFail: {}'.format(test['name']))
            continue
        domain = '.'.join(domain.split('.')[-2:])

        RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST)

    if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict']
                   or test['info_dict']['age_limit'] != 18):
        print('\nPotential missing age_limit check: {}'.format(test['name']))

    elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict']
                         and test['info_dict']['age_limit'] == 18):
        print('\nPotential false negative: {}'.format(test['name']))

    else:
        sys.stdout.write('.')
    sys.stdout.flush()

print()


code:
{{commands}}

complete --command yt-dlp --arguments ":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory"


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import optparse

import yt_dlp
from yt_dlp.utils import shell_quote

FISH_COMPLETION_FILE = 'completions/fish/yt-dlp.fish'
FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in'

EXTRA_ARGS = {
    'remux-video': ['--arguments', 'mp4 mkv', '--exclusive'],
    'recode-video': ['--arguments', 'mp4 flv ogg webm mkv', '--exclusive'],

    # Options that need a file parameter
    'download-archive': ['--require-parameter'],
    'cookies': ['--require-parameter'],
    'load-info': ['--require-parameter'],
    'batch-file': ['--require-parameter'],
}

def build_completion(opt_parser):
    commands = []

    for group in opt_parser.option_groups:
        for option in group.option_list:
            long_option = option.get_opt_string().strip('-')
            complete_cmd = ['complete', '--command', 'yt-dlp', '--long-option', long_option]
            if option._short_opts:
                complete_cmd += ['--short-option', option._short_opts[0].strip('-')]
            if option.help != optparse.SUPPRESS_HELP:
                complete_cmd += ['--description', option.help]
            complete_cmd.extend(EXTRA_ARGS.get(long_option, []))
            commands.append(shell_quote(complete_cmd))

    with open(FISH_COMPLETION_TEMPLATE) as f:
        template = f.read()
    filled_template = template.replace('{{commands}}', '\n'.join(commands))
    with open(FISH_COMPLETION_FILE, 'w') as f:
        f.write(filled_template)

parser = yt_dlp.parseOpts(ignore_config_files=True)[0]
build_completion(parser)


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import codecs
import subprocess

from yt_dlp.aes import aes_encrypt, key_expansion
from yt_dlp.utils import intlist_to_bytes

secret_msg = b'Secret message goes here'

def hex_str(int_list):
    return codecs.encode(intlist_to_bytes(int_list), 'hex')

def openssl_encode(algo, key, iv):
    cmd = ['openssl', 'enc', '-e', '-' + algo, '-K', hex_str(key), '-iv', hex_str(iv)]
    prog = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    out, _ = prog.communicate(secret_msg)
    return out

iv = key = [0x20, 0x15] + 14 * [0]

r = openssl_encode('aes-128-cbc', key, iv)
print('aes_cbc_decrypt')
print(repr(r))

password = key
new_key = aes_encrypt(password, key_expansion(password))
r = openssl_encode('aes-128-ctr', new_key, iv)
print('aes_decrypt_text 16')
print(repr(r))

password = key + 16 * [0]
new_key = aes_encrypt(password, key_expansion(password)) * (32 // 16)
r = openssl_encode('aes-256-ctr', new_key, iv)
print('aes_decrypt_text 32')
print(repr(r))


code:
#!/bin/bash
SERVICE="$1"

if ! command -v pandoc &> /dev/null ; then
  wget -O- https://github.com/jgm/pandoc/releases/download/2.13/pandoc-2.13-linux-amd64.tar.gz | \
      tar -xvzf - --strip-components 1
  export PATH="$PWD/bin/:$PATH"
fi

pdoc() {
  MD="$1"
  HTML="$2"
  TITLE="$3"
  pandoc "$MD" -f gfm-emoji --metadata title="$TITLE" -t html -s -o "$HTML"
}

set -xe

rm -rf public/ || true
git clone --bare https://github.com/nao20010128nao/ytdl-patched.git public/ || \
  git clone --bare https://bitbucket.org/nao20010128nao/ytdl-patched.git public/ || \
  git clone --bare https://forge.tedomum.net/nao20010128nao/ytdl-patched.git public/ || \
  git clone --bare https://gitlab.com/lesmi_the_goodness/ytdl-patched.git public/ || \
  git clone --bare https://gitea.com/nao20010128nao/ytdl-patched.git public/ || \
  git clone --bare https://git.sr.ht/~nao20010128nao/ytdl-patched public/

python devscripts/readme_for_cdn.py README.md README.cdn.md

cd public
pdoc ../README.cdn.md index.html "git clone https://ytdl-patched.${SERVICE}.app/"
tail -n+2 ../docs/supportedsites.md | pdoc - supportedsites.html "List of supported sites by ytdl-patched"
git remote rm origin
git branch -D gh-pages
git reflog expire --expire=now --all
git gc --aggressive --prune=now
mv objects/pack/pack-* . || true
find . -name 'pack-*.pack' -type f -exec bash -c 'git unpack-objects < "$1' _ {} \;
rm -rf pack-* objects/pack/
git update-server-info


code:
#!/bin/bash
cat << EOF > .git/hooks/pre-commit
#!/bin/bash
set -xe
npx lint-staged
make clean
EOF
chmod a+x .git/hooks/pre-commit


code:
# UPX installer for GitHub Actions' Windows
choco install upx

if ($LASTEXITCODE -ne 0) {
    # fallback to ZIP download
    $UpxVersion = "3.96"
    Invoke-WebRequest "https://github.com/upx/upx/releases/download/v${UpxVersion}/upx-${UpxVersion}-win64.zip" -O upx.zip
    Expand-Archive -Path 'upx.zip' -DestinationPath 'upx\'
    # add to PATH as we do for Actions
    echo "$((Get-Item .).FullName)/upx-${UpxVersion}-win64" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
}


code:
import importlib
import random
import re

from ..utils import (
    age_restricted,
    bug_reports_message,
    classproperty,
    get_first_group,
    write_string,
)

# These bloat the lazy_extractors, so allow them to passthrough silently
ALLOWED_CLASSMETHODS = {'get_testcases', 'extract_from_webpage'}
_WARNED = False

class LazyLoadMetaClass(type):
    def __getattr__(cls, name):
        global _WARNED
        if ('_real_class' not in cls.__dict__
                and name not in ALLOWED_CLASSMETHODS and not _WARNED):
            _WARNED = True
            write_string('WARNING: Falling back to normal extractor since lazy extractor '
                         f'{cls.__name__} does not have attribute {name}{bug_reports_message()}\n')
        return getattr(cls.real_class, name)

class LazyLoadExtractor(metaclass=LazyLoadMetaClass):
    @classproperty
    def real_class(cls):
        if '_real_class' not in cls.__dict__:
            cls._real_class = getattr(importlib.import_module(cls._module), cls.__name__)
        return cls._real_class

    def __new__(cls, *args, **kwargs):
        instance = cls.real_class.__new__(cls.real_class)
        instance.__init__(*args, **kwargs)
        return instance


code:
from __future__ import unicode_literals

import subprocess
import os
import re
import sys

# Usage: python3 ./devscripts/make_buildconfig.py key=value ...

values = {
    'git_commit': '',
    'git_upstream_commit': '',
    # only used by Windows executables
    'variant': None,
    # only used by Linux installations, but not on build
    'is_brew': False,
}

for arg in sys.argv[1:]:
    key, value = arg.split('=')
    # Only known key must be used
    if values[key] is None:
        values[key] = value
    else:
        raise Exception(f'Unknown or occupied key: {key}')

# git rev-parse --short ytdlp
sp = subprocess.Popen(
    ['git', 'rev-parse', '--short', 'ytdlp'],
    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
    cwd=os.path.dirname(os.path.abspath(__file__)))
out, err = sp.communicate()
out = out.decode().strip()

if re.match('[0-9a-f]+', out):
    values['git_commit'] = out

# git rev-parse --short yt-dlp
sp = subprocess.Popen(
    ['git', 'rev-parse', '--short', 'yt-dlp'],
    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
    cwd=os.path.dirname(os.path.abspath(__file__)))
out, err = sp.communicate()
out = out.decode().strip()

if re.match('[0-9a-f]+', out):
    values['git_upstream_commit'] = out

pycode = '''# coding: utf-8
# AUTOMATICALLY GENERATED FILE. DO NOT EDIT OR COMMIT.
# Generated by ./devscripts/make_buildconfig.py
from __future__ import unicode_literals

'''

def to_string(o):
    if isinstance(o, str):
        return "'%s'" % o
    elif isinstance(o, bool):
        return f'{o}'
    elif o is None:
        return 'None'
    else:
        raise Exception(f'Unknown object: {o}')

for k, v in values.items():
    pycode += f'{k} = {to_string(v)}\n'

with open('./yt_dlp/build_config.py', 'w') as w:
    w.write(pycode)
print(pycode)


code:
# coding: utf-8
from __future__ import unicode_literals, print_function

import sys
import subprocess

sys.path[:0] = ['.']

from yt_dlp.utils import int_or_none

versions = set()

# https://stackoverflow.com/questions/10649814/get-last-git-tag-from-a-remote-repo-without-cloning
with subprocess.Popen(
        ['git', '-c', 'versionsort.suffix=-', 'ls-remote',
         '--tags', '--sort=v:refname', 'https://chromium.googlesource.com/chromium/src'],
        stdout=subprocess.PIPE,) as proc:
    for line in proc.stdout:
        commit_hash, tag_ref = line.strip().decode().split('\t')
        tag_name = tag_ref[10:]  # trim first "refs/tags/"
        version_tuple = tuple(int_or_none(x) for x in tag_name.split('.') if x.isdigit())
        if len(version_tuple) < 4:
            continue
        versions.add((version_tuple, tag_name))

versions = sorted(versions)
latest_version_major = versions[-1][0][0]
minimum_version = ((latest_version_major - 3, 0, 0, 0), '')  # automatically choose minimum

results = [x[1] for x in versions if x > minimum_version]

lf = '\n'
pycode = f'''# coding: utf-8
# AUTOMATICALLY GENERATED FILE. DO NOT EDIT.
# Generated by ./devscripts/make_chrome_version_list.py
# This list is created from git tags in https://chromium.googlesource.com/chromium/src
from __future__ import unicode_literals

versions = [
{lf.join(f'    "{r}",' for r in sorted(results))}
]

__all__ = ['versions']
'''

with open('./yt_dlp/chrome_versions.py', 'w') as w:
    w.write(pycode)


code:
#!/usr/bin/env python3

import optparse
import re

def main():
    return  # This is unused in yt-dlp

    parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
    options, args = parser.parse_args()
    if len(args) != 2:
        parser.error('Expected an input and an output filename')

    infile, outfile = args

    with open(infile, encoding='utf-8') as inf:
        readme = inf.read()

    bug_text = re.search(
        r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1)
    dev_text = re.search(
        r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING yt-dlp', readme).group(1)

    out = bug_text + dev_text

    with open(outfile, 'w', encoding='utf-8') as outf:
        outf.write(out)

if __name__ == '__main__':
    main()


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import re

from devscripts.utils import (
    get_filename_args,
    read_file,
    read_version,
    write_file,
)

VERBOSE_TMPL = '''
  - type: checkboxes
    id: verbose
    attributes:
      label: Provide verbose output that clearly demonstrates the problem
      options:
        - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
          required: true
        - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
          required: true
  - type: textarea
    id: log
    attributes:
      label: Complete Verbose Output
      description: |
        It should start like this:
      placeholder: |
        [debug] Command-line config: ['-vU', 'test:youtube']
        [debug] Portable config "yt-dlp.conf": ['-i']
        [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
        [debug] yt-dlp version %(version)s [9d339c4] (win32_exe)
        [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
        [debug] Checking exe version: ffmpeg -bsfs
        [debug] Checking exe version: ffprobe -bsfs
        [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
        [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
        [debug] Proxy map: {}
        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
        Latest version: %(version)s, Current version: %(version)s
        yt-dlp is up to date (%(version)s)
        <more lines>
      render: shell
    validations:
      required: true
'''.strip()

NO_SKIP = '''
  - type: checkboxes
    attributes:
      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
      description: Fill all fields even if you think it is irrelevant for the issue
      options:
        - label: I understand that I will be **blocked** if I remove or skip any mandatory\\* field
          required: true
'''.strip()

def main():
    fields = {'version': read_version(), 'no_skip': NO_SKIP}
    fields['verbose'] = VERBOSE_TMPL % fields
    fields['verbose_optional'] = re.sub(r'(\n\s+validations:)?\n\s+required: true', '', fields['verbose'])

    infile, outfile = get_filename_args(has_infile=True)
    write_file(outfile, read_file(infile) % fields)

if __name__ == '__main__':
    main()


code:
#!/usr/bin/env python3

# Allow direct execution
import re
import os
import shutil
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from inspect import getsource

from devscripts.utils import get_filename_args, read_file, write_file

NO_ATTR = object()
STATIC_CLASS_PROPERTIES = [
    'IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_VALID_URL', '_WORKING', '_ENABLED', '_NETRC_MACHINE', 'age_limit'
]
CLASS_METHODS = [
    'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', 'is_suitable'
]
IE_TEMPLATE = '''
class {name}({bases}):
    _module = {module!r}
'''
MODULE_TEMPLATE = read_file('devscripts/lazy_load_template.py')

def main():
    lazy_extractors_filename = get_filename_args(default_outfile='yt_dlp/extractor/lazy_extractors.py')
    if os.path.exists(lazy_extractors_filename):
        os.remove(lazy_extractors_filename)

    _ALL_CLASSES = get_all_ies()  # Must be before import

    from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor

    DummyInfoExtractor = type('InfoExtractor', (InfoExtractor,), {'IE_NAME': NO_ATTR})
    module_src = '\n'.join((
        MODULE_TEMPLATE,
        '    _module = None',
        *extra_ie_code(DummyInfoExtractor),
        '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n',
        *build_ies(_ALL_CLASSES, (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor),
    ))

    write_file(lazy_extractors_filename, f'{module_src}\n')

def get_all_ies():
    PLUGINS_DIRNAME = 'ytdlp_plugins'
    BLOCKED_DIRNAME = f'{PLUGINS_DIRNAME}_blocked'
    if os.path.exists(PLUGINS_DIRNAME):
        # os.rename cannot be used, e.g. in Docker. See https://github.com/yt-dlp/yt-dlp/pull/4958
        shutil.move(PLUGINS_DIRNAME, BLOCKED_DIRNAME)
    try:
        from yt_dlp.extractor.extractors import _ALL_CLASSES
    finally:
        if os.path.exists(BLOCKED_DIRNAME):
            shutil.move(BLOCKED_DIRNAME, PLUGINS_DIRNAME)
    return _ALL_CLASSES

def extra_ie_code(ie, base=None):
    for var in STATIC_CLASS_PROPERTIES:
        val = getattr(ie, var)
        if val != (getattr(base, var) if base else NO_ATTR):
            yield f'    {var} = {val!r}'
    yield ''

    for name in CLASS_METHODS:
        f = getattr(ie, name)
        if not base or f.__func__ != getattr(base, name).__func__:
            yield getsource(f)

def cleanup_regex(regex_str):
    if not isinstance(regex_str, (str, bytes)):
        return regex_str
    has_extended = re.search(r'\(\?[aiLmsux]*x[aiLmsux]*\)', regex_str)  # something like (?xxs) may match, but (?s) or (?i) won't
    if not has_extended:
        return regex_str
    # remove comments
    regex_str = re.sub(r'(?m)\s+#.+?$', '', regex_str)
    # remove spaces and indents
    regex_str = re.sub(r'\s+', '', regex_str)
    # remove x (EXTENDED) from all inline flags
    regex_str = re.sub(r'\(\?([aiLmsux]+)\)', lambda m: '(?%s)' % m.group(1).replace('x', ''), regex_str)
    regex_str = re.sub(r'\(\?\)', '', regex_str)

    return regex_str

def build_ies(ies, bases, attr_base):
    names = []
    for ie in sort_ies(ies, bases):
        yield build_lazy_ie(ie, ie.__name__, attr_base)
        if ie in ies:
            names.append(ie.__name__)

    yield f'\n_ALL_CLASSES = [{", ".join(names)}]'

def sort_ies(ies, ignored_bases):
    """find the correct sorting and add the required base classes so that subclasses can be correctly created"""
    classes, returned_classes = ies[:-1], set()
    assert ies[-1].__name__ == 'GenericIE', 'Last IE must be GenericIE'
    while classes:
        for c in classes[:]:
            bases = set(c.__bases__) - {object, *ignored_bases}
            restart = False
            for b in sorted(bases, key=lambda x: x.__name__):
                if b not in classes and b not in returned_classes:
                    assert b.__name__ != 'GenericIE', 'Cannot inherit from GenericIE'
                    classes.insert(0, b)
                    restart = True
            if restart:
                break
            if bases <= returned_classes:
                yield c
                returned_classes.add(c)
                classes.remove(c)
                break
    yield ies[-1]

def build_lazy_ie(ie, name, attr_base):
    bases = ', '.join({
        'InfoExtractor': 'LazyLoadExtractor',
        'SearchInfoExtractor': 'LazyLoadSearchExtractor',
    }.get(base.__name__, base.__name__) for base in ie.__bases__)

    s = IE_TEMPLATE.format(name=name, module=ie.__module__, bases=bases)
    return s + '\n'.join(extra_ie_code(ie, attr_base))

if __name__ == '__main__':
    main()


code:
#!/usr/bin/env python3

"""
yt-dlp --help | make_readme.py
This must be run in a console of correct width
"""

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import functools
import re

from devscripts.utils import read_file, write_file

README_FILE = 'README.md'

OPTIONS_START = 'General Options:'
OPTIONS_END = 'CONFIGURATION'
EPILOG_START = 'See full documentation'
ALLOWED_OVERSHOOT = 2

DISABLE_PATCH = object()

def take_section(text, start=None, end=None, *, shift=0):
    return text[
        text.index(start) + shift if start else None:
        text.index(end) + shift if end else None
    ]

def apply_patch(text, patch):
    return text if patch[0] is DISABLE_PATCH else re.sub(*patch, text)

options = take_section(sys.stdin.read(), f'\n  {OPTIONS_START}', f'\n{EPILOG_START}', shift=1)

max_width = max(map(len, options.split('\n')))
switch_col_width = len(re.search(r'(?m)^\s{5,}', options).group())
delim = f'\n{" " * switch_col_width}'

PATCHES = (
    (   # Standardize update message
        r'(?m)^(    -U, --update\s+).+(\n    \s.+)*$',
        r'\1Update this program to the latest version',
    ),
    (  # Headings
        r'(?m)^  (\w.+\n)(    (?=\w))?',
        r'## \1'
    ),
    (  # Do not split URLs
        rf'({delim[:-1]})? (?P<label>\[\S+\] )?(?P<url>https?({delim})?:({delim})?/({delim})?/(({delim})?\S+)+)\s',
        lambda mobj: ''.join((delim, mobj.group('label') or '', re.sub(r'\s+', '', mobj.group('url')), '\n'))
    ),
    (  # Do not split "words"
        rf'(?m)({delim}\S+)+$',
        lambda mobj: ''.join((delim, mobj.group(0).replace(delim, '')))
    ),
    (  # Allow overshooting last line
        rf'(?m)^(?P<prev>.+)${delim}(?P<current>.+)$(?!{delim})',
        lambda mobj: (mobj.group().replace(delim, ' ')
                      if len(mobj.group()) - len(delim) + 1 <= max_width + ALLOWED_OVERSHOOT
                      else mobj.group())
    ),
    (  # Avoid newline when a space is available b/w switch and description
        DISABLE_PATCH,  # This creates issues with prepare_manpage
        r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim),
        r'\1 '
    ),
)

readme = read_file(README_FILE)

write_file(README_FILE, ''.join((
    take_section(readme, end=f'## {OPTIONS_START}'),
    functools.reduce(apply_patch, PATCHES, options),
    take_section(readme, f'# {OPTIONS_END}'),
)))


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from devscripts.utils import get_filename_args, write_file
from yt_dlp.extractor import list_extractor_classes

def main():
    out = '\n'.join(ie.description() for ie in list_extractor_classes() if ie.IE_DESC is not False)
    write_file(get_filename_args(), f'# Supported sites\n{out}\n')

if __name__ == '__main__':
    main()


code:
from __future__ import unicode_literals

import os
import shutil
import subprocess
import sys
import optparse
import datetime
import time

sys.path[:0] = ['.']

from yt_dlp.utils import check_executable

try:
    iterations = str(int(os.environ['ZOPFLI_ITERATIONS']))
except BaseException:
    iterations = '30'

parser = optparse.OptionParser(usage='%prog PYTHON')
options, args = parser.parse_args()
if len(args) != 1:
    parser.error('Expected python executable name for shebang')

PYTHON = args[0]

# 200001010101
date = datetime.datetime(year=2000, month=1, day=1, hour=1, minute=1, second=1)
modTime = time.mktime(date.timetuple())

try:
    shutil.rmtree('zip/')
except FileNotFoundError:
    pass
os.makedirs('zip/', exist_ok=True)

files = [(dir, file) for (dir, _, c) in os.walk('yt_dlp') for file in c if file.endswith('.py')]
all_paths = []

for (dir, file) in files:
    joined = os.path.join(dir, file)
    all_paths.append(joined)
    dest = os.path.join('zip', joined)
    os.makedirs(os.path.join('zip', dir), exist_ok=True)
    shutil.copy(joined, dest, follow_symlinks=False)
    os.utime(dest, (modTime, modTime))

os.rename('zip/yt_dlp/__main__.py', 'zip/__main__.py')
all_paths.remove('yt_dlp/__main__.py')
all_paths[:0] = ['__main__.py']

if check_executable('7z', []):
    ret = subprocess.Popen(
        ['7z', 'a', '-mm=Deflate', '-mfb=258', '-mpass=15', '-mtc-', '../ytdl-patched.zip'] + all_paths,
        cwd='zip/').wait()
elif check_executable('zip', ['-h']):
    ret = subprocess.Popen(
        ['zip', '-9', '../ytdl-patched.zip'] + all_paths,
        cwd='zip/').wait()
else:
    raise Exception('Cannot find ZIP archiver')

if ret != 0:
    raise Exception('ZIP archiver returned error: %d' % ret)

if check_executable('advzip', []):
    subprocess.Popen(
        ['advzip', '-z', '-4', '-i', iterations, 'ytdl-patched.zip']).wait()

shutil.rmtree('zip/')

with open('ytdl-patched', 'wb') as ytdl:
    ytdl.write(b'#!%s\n' % PYTHON.encode('utf8'))
    with open('ytdl-patched.zip', 'rb') as zip:
        ytdl.write(zip.read())

os.remove('ytdl-patched.zip')

os.chmod('ytdl-patched', 0o755)


code:
#!/usr/bin/env python3
# coding: utf-8
from __future__ import unicode_literals

import sys

# this script converts something like 2021.03.24 into 2021.3.24
print('.'.join(str(int(x)) for x in sys.argv[1].split('.')))


code:
#!/usr/bin/env python3

# A script to parse options and return dict representation of it
# usage: ./devscripts/parse-args.py <options>
# non-option values such as URLs are ignored

import sys
import os
import pprint
sys.path[:0] = ['.', 'devscripts']

from yt_dlp import parse_options

if os.getenv('FULL_DICT') == 'y':
    none_opts = {}
else:
    _, _, _, none_opts = parse_options([''], ignore_config_files=True)

_, _, _, parsed = parse_options(ignore_config_files=True)

def erase_empty_values(dct):
    for k in set(k for k, v in dct.items() if not v):
        dct.pop(k, None)

for k, nv in none_opts.items():
    if k not in parsed:
        continue
    pv = parsed[k]
    if k == 'outtmpl':
        erase_empty_values(nv)
        erase_empty_values(pv)
    if pv == nv:
        parsed.pop(k, None)

pprint.pprint(parsed)


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import os.path
import re

from devscripts.utils import (
    compose_functions,
    get_filename_args,
    read_file,
    write_file,
)

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
README_FILE = os.path.join(ROOT_DIR, 'README.md')

PREFIX = r'''%yt-dlp(1)

# NAME

yt\-dlp \- A youtube-dl fork with additional features and patches

# SYNOPSIS

**yt-dlp** \[OPTIONS\] URL [URL...]

# DESCRIPTION

'''

def filter_excluded_sections(readme):
    EXCLUDED_SECTION_BEGIN_STRING = re.escape('<!-- MANPAGE: BEGIN EXCLUDED SECTION -->')
    EXCLUDED_SECTION_END_STRING = re.escape('<!-- MANPAGE: END EXCLUDED SECTION -->')
    readme = re.sub(
        rf'(?s){EXCLUDED_SECTION_BEGIN_STRING}.+?{EXCLUDED_SECTION_END_STRING}\n',
        '', readme)

    EXCLUDED_MARKER_BEGIN_STRING = re.escape('<!-- MARKER BEGIN -->')
    EXCLUDED_MARKER_END_STRING = re.escape('<!-- MARKER END -->')
    return re.sub(
        rf'(?s){EXCLUDED_MARKER_BEGIN_STRING}.+?{EXCLUDED_MARKER_END_STRING}\n',
        '', readme)

def move_sections(readme):
    MOVE_TAG_TEMPLATE = '<!-- MANPAGE: MOVE "%s" SECTION HERE -->'
    sections = re.findall(r'(?m)^%s$' % (
        re.escape(MOVE_TAG_TEMPLATE).replace(r'\%', '%') % '(.+)'), readme)

    for section_name in sections:
        move_tag = MOVE_TAG_TEMPLATE % section_name
        if readme.count(move_tag) > 1:
            raise Exception(f'There is more than one occurrence of "{move_tag}". This is unexpected')

        sections = re.findall(rf'(?sm)(^# {re.escape(section_name)}.+?)(?=^# )', readme)
        if len(sections) < 1:
            raise Exception(f'The section {section_name} does not exist')
        elif len(sections) > 1:
            raise Exception(f'There are multiple occurrences of section {section_name}, this is unhandled')

        readme = readme.replace(sections[0], '', 1).replace(move_tag, sections[0], 1)
    return readme

def filter_options(readme):
    section = re.search(r'(?sm)^# USAGE AND OPTIONS\n.+?(?=^# )', readme).group(0)
    options = '# OPTIONS\n'
    for line in section.split('\n')[1:]:
        mobj = re.fullmatch(r'''(?x)
                \s{4}(?P<opt>-(?:,\s|[^\s])+)
                (?:\s(?P<meta>(?:[^\s]|\s(?!\s))+))?
                (\s{2,}(?P<desc>.+))?
            ''', line)
        if not mobj:
            options += f'{line.lstrip()}\n'
            continue
        option, metavar, description = mobj.group('opt', 'meta', 'desc')

        # Pandoc's definition_lists. See http://pandoc.org/README.html
        option = f'{option} *{metavar}*' if metavar else option
        description = f'{description}\n' if description else ''
        options += f'\n{option}\n:   {description}'
        continue

    return readme.replace(section, options, 1)

TRANSFORM = compose_functions(filter_excluded_sections, move_sections, filter_options)

def main():
    write_file(get_filename_args(), PREFIX + TRANSFORM(read_file(README_FILE)))

if __name__ == '__main__':
    main()


code:
# coding: utf-8
from __future__ import unicode_literals

from PyInstaller import __main__

import zlib
import zopfli
import os

try:
    iterations = int(os.environ['ZOPFLI_ITERATIONS'])
except BaseException:
    iterations = 30

def zlib_compress(data, level=-1):
    c = zopfli.ZopfliCompressor(zopfli.ZOPFLI_FORMAT_ZLIB, iterations=iterations)
    return c.compress(data) + c.flush()

zlib.compress = zlib_compress

__main__.run()


code:
from __future__ import unicode_literals

import re
import sys
import subprocess
import os

infile, outfile = sys.argv[1:]

# usage: python3 devscripts/readme_for_cdn.py ../README.md to_be_converted.md

# git rev-parse --short master
git_commit = ''
for cwd in [
    os.path.join(os.getcwd(), os.path.abspath(__file__), '../public'),
    os.path.join(os.getcwd(), os.path.abspath(__file__), 'public'),
    os.path.dirname(os.path.abspath(__file__)),
]:
    try:
        cwd = os.path.normpath(cwd)
        sp = subprocess.Popen(
            ['git', 'rev-parse', '--short', 'master'],
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            cwd=cwd)
        out, _ = sp.communicate()
        out = out.decode().strip()
        if re.match('[0-9a-f]+', out):
            git_commit = out
            break
    except BaseException:
        pass

if not git_commit:
    git_commit = os.environ.get('VERCEL_GIT_COMMIT_SHA')

if isinstance(git_commit, str):
    if len(git_commit) > 8:
        git_commit = git_commit[0:8]
else:
    git_commit = 'master'

# https://vercel.com/docs/cli#commands/overview/unique-options

information_section = [
    '- current commit: [`%(git_commit)s`](https://github.com/nao20010128nao/ytdl-patched/commit/%(git_commit)s)' % {'git_commit': git_commit},
    '- [see list of supported sites](/supportedsites.html)',
]

release_tag = os.environ.get('GITHUB_RELEASE_TAG')

if not release_tag:
    sys.path[:0] = ['.']

    from youtube_dl.extractor.common import InfoExtractor
    from test.helper import FakeYDL

    class TestIE(InfoExtractor):
        pass

    ie = TestIE(FakeYDL({'verbose': False}))
    script_id = 'readme_for_cdn'

    data = ie._download_json(
        'https://api.github.com/repos/nao20010128nao/ytdl-patched/releases/latest',
        script_id, note=False)
    release_tag = data['tag_name']

if release_tag:
    information_section.append('- [download ytdl-patched](https://github.com/nao20010128nao/ytdl-patched/releases/tag/%s)' % release_tag)
    information_section.append('  - [for Linux/macOS](https://github.com/nao20010128nao/ytdl-patched/releases/download/%s/youtube-dl)' % release_tag)
    information_section.append('  - [for Windows](https://github.com/nao20010128nao/ytdl-patched/releases/download/%s/youtube-dl-red.exe)' % release_tag)
    information_section.append('  - [for pip](https://github.com/nao20010128nao/ytdl-patched/releases/download/%s/youtube-dl.tar.gz)' % release_tag)
    information_section.append('  - or by Homebrew: `brew install nao20010128nao/my/ytdl-patched`')

MARKER_RE = r'(?m)^<!-- MARKER BEGIN -->[^\0]+<!-- MARKER END -->$'

NAVIGATE_TXT = """
# Information

%s
""" % ('\n'.join(information_section))

markdown = ''

with open(infile, 'r') as r:
    markdown = r.read()

if isinstance(markdown, bytes):
    markdown = markdown.decode('utf-8')

markdown = re.sub(MARKER_RE, NAVIGATE_TXT, markdown)

if sys.version_info < (3, ):
    markdown = markdown.encode('utf-8')

with open(outfile, 'w') as w:
    w.write(markdown)


code:
#!/usr/bin/env python3
# coding: utf-8
from __future__ import unicode_literals

import sys
import re

# https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/
# https://docs.github.com/en/actions/guides/about-continuous-integration#skipping-workflow-runs
skips = [
    '[ci skip]', '[skip ci]',
    '[no ci]',
    '[skip actions]', '[actions skip]',
    'skip-checks: true', 'skip-checks:true']

compiled = [re.compile(re.escape(x)) for x in skips]
removed = 0

def count_removed(m):
    global removed
    removed += 1
    return ''

for file in sys.argv[1:]:
    with open(file, 'r') as f:
        content = f.read()
    for reg in compiled:
        content = re.sub(reg, count_removed, content)
    with open(file, 'w') as f:
        f.write(content)

print('%d occurrences removed' % removed)


code:
#!/bin/bash
set -x
for I in "$@"; do
  wget https://github.com/ytdl-patched/ytdl-patched/raw/ytdlp/"$I" -O "$I"
done


code:
#!/bin/bash
set -x
for I in "$@"; do
  wget https://github.com/yt-dlp/yt-dlp/raw/master/"$I" -O "$I"
done


code:
@setlocal
@echo off
cd /d %~dp0..

if ["%~1"]==[""] (
    set "test_set="test""
) else if ["%~1"]==["core"] (
    set "test_set="-m not download""
) else if ["%~1"]==["download"] (
    set "test_set="-m "download""
) else (
    echo.Invalid test type "%~1". Use "core" ^| "download"
    exit /b 1
)

set PYTHONWARNINGS=error
pytest %test_set%


code:
#!/usr/bin/env sh

if [ -z "$1" ]; then
    test_set='test'
elif [ "$1" = 'core' ]; then
    test_set="-m not download"
elif [ "$1" = 'download' ]; then
    test_set="-m download"
else
    echo 'Invalid test type "'"$1"'". Use "core" | "download"'
    exit 1
fi

python3 -bb -Werror -m pytest "$test_set"


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from yt_dlp.utils import ExtractorError, traverse_obj
from yt_dlp.extractor.common import InfoExtractor
from test.helper import FakeYDL

class TestIE(InfoExtractor):
    pass

ie = TestIE(FakeYDL({
    'verbose': False,
    'socket_timeout': 120,
}))

orig_dwh = ie._download_webpage_handle

dwh_args = orig_dwh.__code__.co_varnames[1:orig_dwh.__code__.co_argcount]

def retry_download_webpage_handle(*args, **kw):
    kw = {
        **dict(zip(dwh_args, args)),
        **kw,
    }
    retries = 3
    fatal = kw.get('fatal', True)
    note = kw.get('note')
    kw['fatal'] = True
    for i in range(retries + 1):
        try:
            if i:
                kw['note'] = f'{note} (retry {i} of {retries})'
            return orig_dwh(**kw)
        except ExtractorError as e:
            if retries == i + 1:
                if fatal:
                    raise
                else:
                    ie.report_warning(e)
                    break
            ie.report_warning(f'{e} Retrying...')

ie._download_webpage_handle = retry_download_webpage_handle

def sanitize_hostname(hostname):
    # trim trailing slashes
    hostname = re.sub(r'[/\\]+$', '', hostname)
    # trim port number
    hostname = re.sub(r':\d+$', '', hostname)
    return hostname

def traverse_sanitize(*arg, **kw):
    return map(sanitize_hostname, traverse_obj(*arg, **kw) or [])


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import argparse
import functools
import re

from devscripts.utils import compose_functions, read_file, write_file

VERSION_FILE = 'yt_dlp/version.py'

def parse_options():
    parser = argparse.ArgumentParser(description='Set the build variant of the package')
    parser.add_argument('variant', help='Name of the variant')
    parser.add_argument('-M', '--update-message', default=None, help='Message to show in -U')
    return parser.parse_args()

def property_setter(name, value):
    return functools.partial(re.sub, rf'(?m)^{name}\s*=\s*.+$', f'{name} = {value!r}')

opts = parse_options()
transform = compose_functions(
    property_setter('VARIANT', opts.variant),
    property_setter('UPDATE_HINT', opts.update_message)
)

write_file(VERSION_FILE, transform(read_file(VERSION_FILE)))


code:
commit be7be5da2a823ac2ce8826c695c6a56afdc17b33
Author: nao20010128nao <nao20010128@gmail.com>
Date:   Sat Jul 31 06:44:58 2021 +0900

    plugin.api.http_session: do not fail even if timeout is not set

diff --git a/src/streamlink/plugin/api/http_session.py b/src/streamlink/plugin/api/http_session.py
index 6cd3764..18bdaa8 100644
--- a/src/streamlink/plugin/api/http_session.py
+++ b/src/streamlink/plugin/api/http_session.py
@@ -134,7 +134,7 @@ class HTTPSession(Session):
         raise_for_status = kwargs.pop("raise_for_status", True)
         schema = kwargs.pop("schema", None)
         session = kwargs.pop("session", None)
-        timeout = kwargs.pop("timeout", self.timeout)
+        timeout = kwargs.pop("timeout", getattr(self, 'timeout', None))
         total_retries = kwargs.pop("retries", 0)
         retry_backoff = kwargs.pop("retry_backoff", 0.3)
         retry_max_backoff = kwargs.pop("retry_max_backoff", 10.0)


code:
#!/bin/bash
# an convenient script to easily trigger "Merge upstream" workflow
exec gh workflow run -R ytdl-patched/ytdl-patched merge.yml


code:
#!/usr/bin/env python3

"""
Usage: python3 ./devscripts/update-formulae.py <path-to-formulae-rb> <version>
version can be either 0-aligned (yt-dlp version) or normalized (PyPi version)
"""

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import re
from devscripts.utils import read_file, write_file

filename, sha256sum, version, url = sys.argv[1:]

normalized_version = '.'.join(str(int(x)) for x in version.split('.'))

formulae_text = read_file(filename)

formulae_text = re.sub(r'sha256 "[0-9a-f]*?" # replace-marker', 'sha256 "%s" # replace-marker' % sha256sum, formulae_text)
formulae_text = re.sub(r'version "[^"]*?" # replace-marker', 'version "%s" # replace-marker' % version, formulae_text)
formulae_text = re.sub(r'url "[^"]*?" # replace-marker', 'url "%s" # replace-marker' % url, formulae_text)

write_file(filename, formulae_text)


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import contextlib
import subprocess
import sys
from datetime import datetime

from devscripts.utils import read_version, write_file

def get_new_version(revision):
    time = datetime.utcnow()
    version = time.strftime('%Y.%m.%d.%s')

    if revision:
        assert revision.isdigit(), 'Revision must be a number'
    else:
        old_version = read_version().split('.')
        if version.split('.') == old_version[:3]:
            revision = str(int((old_version + [0])[3]) + 1)

    return (f'{version}.{revision}' if revision else version), time.strftime('%s')

def get_git_head():
    with contextlib.suppress(Exception):
        sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE)
        return sp.communicate()[0].decode().strip() or None

VERSION, UNIX_TIME = get_new_version((sys.argv + [''])[1])
normalized_version = '.'.join(str(int(x)) for x in VERSION.split('.'))
GIT_HEAD = get_git_head()

VERSION_FILE = f'''\
# Autogenerated by devscripts/update-version.py

__version__ = {VERSION!r}

RELEASE_GIT_HEAD = {GIT_HEAD!r}

VARIANT = None

UPDATE_HINT = None
'''

write_file('yt_dlp/version.py', VERSION_FILE)
print(f'::set-output name=ytdlp_version::{VERSION}')
print(f'::set-output name=latest_version::{VERSION}')
print(f'::set-output name=latest_version_normalized::{normalized_version}')
print(f'::set-output name=latest_version_numeric::{UNIX_TIME}')
print(f'\nVersion = {VERSION}, Git HEAD = {GIT_HEAD}')


code:
import argparse
import functools

def read_file(fname):
    with open(fname, encoding='utf-8') as f:
        return f.read()

def write_file(fname, content):
    with open(fname, 'w', encoding='utf-8') as f:
        return f.write(content)

# Get the version without importing the package
def read_version(fname='yt_dlp/version.py'):
    exec(compile(read_file(fname), fname, 'exec'))
    return locals()['__version__']

def get_filename_args(has_infile=False, default_outfile=None):
    parser = argparse.ArgumentParser()
    if has_infile:
        parser.add_argument('infile', help='Input file')
    kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {}
    parser.add_argument('outfile', **kwargs, help='Output file')

    opts = parser.parse_args()
    if has_infile:
        return opts.infile, opts.outfile
    return opts.outfile

def compose_functions(*functions):
    return lambda x: functools.reduce(lambda y, f: f(y), functions, x)


code:
#!/bin/bash
export __DIRNAME="$(basename "$(pwd)")"

if [[ "$__DIRNAME" == "vercel" ]] ; then
  cd ../../
fi

yum install -y wget tar gzip

./devscripts/git-dumb-cdn.sh vercel

if [[ "$__DIRNAME" == "vercel" ]] ; then
  cp -r public/* devscripts/vercel/
fi


code:
#compdef yt-dlp

__yt_dlp() {
    local curcontext="$curcontext" fileopts diropts cur prev
    typeset -A opt_args
    fileopts="{{fileopts}}"
    diropts="{{diropts}}"
    cur=$words[CURRENT]
    case $cur in
        :)
            _arguments '*: :(::ytfavorites ::ytrecommended ::ytsubscriptions ::ytwatchlater ::ythistory)'
        ;;
        *)
            prev=$words[CURRENT-1]
            if [[ ${prev} =~ ${fileopts} ]]; then
                _path_files
            elif [[ ${prev} =~ ${diropts} ]]; then
                _path_files -/
            elif [[ ${prev} == "--remux-video" ]]; then
                _arguments '*: :(mp4 mkv)'
            elif [[ ${prev} == "--recode-video" ]]; then
                _arguments '*: :(mp4 flv ogg webm mkv)'
            else
                _arguments '*: :({{flags}})'
            fi
        ;;
    esac
}

__yt_dlp


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import yt_dlp

ZSH_COMPLETION_FILE = "completions/zsh/_yt-dlp"
ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in"

def build_completion(opt_parser):
    opts = [opt for group in opt_parser.option_groups
            for opt in group.option_list]
    opts_file = [opt for opt in opts if opt.metavar == "FILE"]
    opts_dir = [opt for opt in opts if opt.metavar == "DIR"]

    fileopts = []
    for opt in opts_file:
        if opt._short_opts:
            fileopts.extend(opt._short_opts)
        if opt._long_opts:
            fileopts.extend(opt._long_opts)

    diropts = []
    for opt in opts_dir:
        if opt._short_opts:
            diropts.extend(opt._short_opts)
        if opt._long_opts:
            diropts.extend(opt._long_opts)

    flags = [opt.get_opt_string() for opt in opts]

    with open(ZSH_COMPLETION_TEMPLATE) as f:
        template = f.read()

    template = template.replace("{{fileopts}}", "|".join(fileopts))
    template = template.replace("{{diropts}}", "|".join(diropts))
    template = template.replace("{{flags}}", " ".join(flags))

    with open(ZSH_COMPLETION_FILE, "w") as f:
        f.write(template)

parser = yt_dlp.parseOpts(ignore_config_files=True)[0]
build_completion(parser)


code:
FROM mcr.microsoft.com/vscode/devcontainers/universal:linux

ENV PATH="$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$PATH"

ARG NODE_VERSION="14"
RUN ( umask 0002 && . $HOME/.nvm/nvm.sh && nvm install ${NODE_VERSION} || true ) && \
    sed -i 's/# export LANG/export LANG/' ~/.bashrc && \
    pip3 install --user -U pytest nose flake8 pip && \
    sudo add-apt-repository -y ppa:git-core/ppa && \
    sudo apt update && \
    sudo apt upgrade -y && \
    sudo apt install -y --no-install-recommends \
        bzip2 ca-certificates curl file fonts-dejavu-core g++ \
        less libz-dev locales make netbase openssh-client patch \
        uuid-runtime tzdata ffmpeg rtmpdump shellcheck pandoc \
        aria2 transmission-cli axel httpie && \
    sudo rm -rf /var/lib/apt/lists/* && \
    sudo mkdir -p /home/linuxbrew/.linuxbrew && \
    sudo chown -R codespace:codespace /home/linuxbrew/ && \
    git clone https://github.com/Homebrew/brew /home/linuxbrew/.linuxbrew/ && \
    brew doctor && \
    echo "eval \$($(brew --prefix)/bin/brew shellenv)" >> ~/.bash_profile && \
    HOMEBREW_NO_ANALYTICS=1 HOMEBREW_NO_AUTO_UPDATE=1 brew tap homebrew/core nao20010128nao/my && \
    brew install nao20010128nao/my/advcomp docker-clean python@3.7 python@3.8 python@3.9 atomicparsley && \
    brew cleanup --prune 0 && \
    { git -C /home/linuxbrew/.linuxbrew config --unset gc.auto; true; } && \
    { git -C /home/linuxbrew/.linuxbrew config --unset homebrew.devcmdrun; true; } && \
    rm -rf ~/.cache && \
    chmod -R g+w,o-w /home/linuxbrew/.linuxbrew && \
    git config --global pull.rebase false && \
    wget https://github.com/yt-dlp/SponSkrub/releases/download/2021.03.15/sponskrub -O ~/.local/bin/sponskrub


code:
ARG base_tag=latest
FROM python:${base_tag}

RUN mkdir -p /opt/bin
ENV PATH="$PATH:/opt/bin"

ADD artifacts/youtube-dl /opt/bin/

RUN chmod a+x /opt/bin/youtube-dl && \
    youtube-dl --version && youtube-dl --help


code:
ARG base_tag=windowsservercore-1806
FROM python:${base_tag}

RUN powershell New-Item -ItemType "directory" -Path C:/Users/ContainerAdministrator/AppData/Local -Name youtube-dl
ADD artifacts/youtube-dl.exe C:/Users/ContainerAdministrator/AppData/Local/youtube-dl
# https://stackoverflow.com/questions/42092932/appending-to-path-in-a-windows-docker-container
RUN setx path "C:/Users/ContainerAdministrator/AppData/Local/youtube-dl;$env:path"

RUN youtube-dl --version ; youtube-dl --help


code:
owner: ytdl-patched
repo: ytdl-patched
description: yt-dlp fork with some more features
homepage: https://github.com/yt-dlp/yt-dlp


code:
#!/bin/bash
set -xe
# https://unix.stackexchange.com/questions/89275/how-to-create-ico-file-with-more-than-one-image
convert -background transparent "$1" -define icon:auto-resize=16,32,48,64,128,192,256 "${1%%.*}.ico"


code:
#!/usr/bin/env python3

# Allow direct execution
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

import platform

from PyInstaller.__main__ import run as run_pyinstaller

from devscripts.utils import read_version

import zlib
import zopfli

try:
    iterations = int(os.environ['ZOPFLI_ITERATIONS'])
except BaseException:
    iterations = 30

def zlib_compress(data, level=-1):
    c = zopfli.ZopfliCompressor(zopfli.ZOPFLI_FORMAT_ZLIB, iterations=iterations)
    return c.compress(data) + c.flush()

zlib.compress = zlib_compress

OS_NAME, MACHINE, ARCH = sys.platform, platform.machine(), platform.architecture()[0][:2]
if MACHINE in ('x86_64', 'AMD64') or ('i' in MACHINE and '86' in MACHINE):
    # NB: Windows x86 has MACHINE = AMD64 irrespective of bitness
    MACHINE = 'x86' if ARCH == '32' else ''

ICON = os.getenv('windows_icon') or 'red'

def main():
    opts, version = parse_options(), read_version()

    onedir = '--onedir' in opts or '-D' in opts
    if not onedir and '-F' not in opts and '--onefile' not in opts:
        opts.append('--onefile')

    name, final_file = exe(onedir)
    print(f'Building ytdl-patched v{version} {ARCH}bit for {OS_NAME} with options {opts}')
    print('Remember to update the version using  "devscripts/update-version.py"')
    if not os.path.isfile('yt_dlp/extractor/lazy_extractors.py'):
        print('WARNING: Building without lazy_extractors. Run  '
              '"devscripts/make_lazy_extractors.py"  to build lazy extractors', file=sys.stderr)
    print(f'Destination: {final_file}\n')

    opts = [
        f'--name={name}',
        f'--icon=icons/youtube_social_squircle_{ICON}.ico',
        '--upx-exclude=vcruntime140.dll',
        '--noconfirm',
        # NB: Modules that are only imported dynamically must be added here.
        # --collect-submodules may not work correctly if user has a yt-dlp installed via PIP
        '--hidden-import=yt_dlp.compat._legacy',
        *dependency_options(),
        *opts,
        'yt_dlp/__main__.py',
    ]

    print(f'Running PyInstaller with {opts}')
    run_pyinstaller(opts)
    set_version_info(final_file, version)

def parse_options():
    # Compatibility with older arguments
    opts = sys.argv[1:]
    if opts[0:1] in (['32'], ['64']):
        if ARCH != opts[0]:
            raise Exception(f'{opts[0]}bit executable cannot be built on a {ARCH}bit system')
        opts = opts[1:]
    return opts

def exe(onedir):
    """@returns (name, path)"""
    name = '_'.join(filter(None, (
        'ytdl-patched',
        OS_NAME == 'darwin' and 'macos',
        ARCH == '32' and 'x86'
    )))
    return name, ''.join(filter(None, (
        'dist/',
        onedir and f'{name}/',
        name,
        OS_NAME == 'win32' and '.exe'
    )))

def version_to_list(version):
    version_list = version.split('.')
    return list(map(int, version_list)) + [0] * (4 - len(version_list))

def dependency_options():
    # Due to the current implementation, these are auto-detected, but explicitly add them just in case
    dependencies = [pycryptodome_module(), 'mutagen', 'brotli', 'certifi', 'websockets']
    excluded_modules = ('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts')

    yield from (f'--hidden-import={module}' for module in dependencies)
    yield '--collect-submodules=websockets'
    yield from (f'--exclude-module={module}' for module in excluded_modules)

def pycryptodome_module():
    try:
        import Cryptodome  # noqa: F401
    except ImportError:
        try:
            import Crypto  # noqa: F401
            print('WARNING: Using Crypto since Cryptodome is not available. '
                  'Install with: pip install pycryptodomex', file=sys.stderr)
            return 'Crypto'
        except ImportError:
            pass
    return 'Cryptodome'

def set_version_info(exe, version):
    if OS_NAME == 'win32':
        windows_set_version(exe, version)

def windows_set_version(exe, version):
    from PyInstaller.utils.win32.versioninfo import (
        FixedFileInfo,
        SetVersion,
        StringFileInfo,
        StringStruct,
        StringTable,
        VarFileInfo,
        VarStruct,
        VSVersionInfo,
    )

    version_list = version_to_list(version)
    suffix = '_x86' if ARCH == '32' else ''
    SetVersion(exe, VSVersionInfo(
        ffi=FixedFileInfo(
            filevers=version_list,
            prodvers=version_list,
            mask=0x3F,
            flags=0x0,
            OS=0x4,
            fileType=0x1,
            subtype=0x0,
            date=(0, 0),
        ),
        kids=[
            StringFileInfo([StringTable('040904B0', [
                StringStruct('Comments', 'ytdl-patched%s Command Line Interface.' % suffix),
                StringStruct('CompanyName', 'https://github.com/ytdl-patched'),
                StringStruct('FileDescription', 'ytdl-patched%s' % (' (32 Bit)' if ARCH == '32' else '')),
                StringStruct('FileVersion', version),
                StringStruct('InternalName', f'ytdl-patched{suffix}'),
                StringStruct('LegalCopyright', 'nao20010128gmail.com | UNLICENSE'),
                StringStruct('OriginalFilename', f'ytdl-patched{suffix}.exe'),
                StringStruct('ProductName', f'ytdl-patched{suffix}'),
                StringStruct(
                    'ProductVersion', f'{version}{suffix} on Python {platform.python_version()}'),
            ])]), VarFileInfo([VarStruct('Translation', [0, 1200])])
        ]
    ))

if __name__ == '__main__':
    main()


code:
mutagen
pycryptodomex
websockets
brotli; platform_python_implementation=='CPython'
brotlicffi; platform_python_implementation!='CPython'
certifi


code:
[wheel]
universal = true

[flake8]
exclude = build,venv,.tox,.git,.pytest_cache
ignore = E402,E501,E731,E741,W503
max_line_length = 120
per_file_ignores =
    devscripts/lazy_load_template.py: F401

[autoflake]
ignore-init-module-imports = true
ignore-pass-after-docstring = true
remove-all-unused-imports = true
remove-duplicate-keys = true
remove-unused-variables = true

[tool:pytest]
addopts = -ra -v --strict-markers
markers =
    download

[tox:tox]
skipsdist = true
envlist = py{36,37,38,39,310},pypy{36,37,38,39}
skip_missing_interpreters = true

[testenv]  # tox
deps =
   pytest
commands = pytest {posargs:"-m not download"}
passenv = HOME  # For test_compat_expanduser
setenv =
    # PYTHONWARNINGS = error  # Catches PIP's warnings too

[isort]
py_version = 37
multi_line_output = VERTICAL_HANGING_INDENT
line_length = 80
reverse_relative = true
ensure_newline_before_comments = true
include_trailing_comma = true
known_first_party =
    test


code:
#!/usr/bin/env python3
import os
import os.path
import subprocess
import sys
import warnings

try:
    from setuptools import Command, find_packages, setup
    setuptools_available = True
except ImportError:
    from distutils.core import Command, setup
    setuptools_available = False

from devscripts.utils import read_file, read_version

VERSION = read_version()

DESCRIPTION = 'A youtube-dl fork with additional features and patches'

LONG_DESCRIPTION = '\n\n'.join((
    'Official repository: <https://github.com/yt-dlp/yt-dlp>',
    '**PS**: Some links in this document will not work since this is a copy of the README.md from Github',
    read_file('README.md')))

REQUIREMENTS = read_file('requirements.txt').splitlines()

def packages():
    if setuptools_available:
        return find_packages(exclude=('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts'))

    return [
        'yt_dlp', 'yt_dlp.extractor',
        'yt_dlp.compat', 'yt_dlp.downloader',
        'yt_dlp.websocket', 'yt_dlp.postprocessor',
        'yt_dlp.extractor.peertube', 'yt_dlp.extractor.misskey',
        'yt_dlp.extractor.mastodon',
        'yt_dlp.extractor.anvato_token_generator'
    ]

def py2exe_params():
    import py2exe  # noqa: F401

    warnings.warn(
        'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
        'The recommended way is to use "pyinst.py" to build using pyinstaller')

    return {
        'console': [{
            'script': './yt_dlp/__main__.py',
            'dest_base': 'ytdl-patched',
            'version': VERSION,
            'description': DESCRIPTION,
            'comments': LONG_DESCRIPTION.split('\n')[0],
            'product_name': 'ytdl-patched',
            'product_version': VERSION,
            'icon_resources': [(1, 'devscripts/logo.ico')],
        }],
        'options': {
            'py2exe': {
                'bundle_files': 0,
                'compressed': 1,
                'optimize': 2,
                'dist_dir': './dist',
                'excludes': ['Crypto', 'Cryptodome'],  # py2exe cannot import Crypto
                'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
                # Modules that are only imported dynamically must be added here
                'includes': ['yt_dlp.compat._legacy'],
            }
        },
        'zipfile': None
    }

def build_params():
    files_spec = [
        ('share/bash-completion/completions', ['completions/bash/ytdl-patched']),
        ('share/zsh/site-functions', ['completions/zsh/_ytdl-patched']),
        ('share/fish/vendor_completions.d', ['completions/fish/ytdl-patched.fish']),
        ('share/doc/yt_dlp', ['README.txt']),
        ('share/man/man1', ['ytdl-patched.1'])
    ]
    data_files = []
    for dirname, files in files_spec:
        resfiles = []
        for fn in files:
            if not os.path.exists(fn):
                warnings.warn(f'Skipping file {fn} since it is not present. Try running " make pypi-files " first')
            else:
                resfiles.append(fn)
        data_files.append((dirname, resfiles))

    params = {'data_files': data_files}

    if setuptools_available:
        params['entry_points'] = {'console_scripts': ['ytdl-patched = yt_dlp:main']}
    else:
        params['scripts'] = ['ytdl-patched']
    return params

class build_lazy_extractors(Command):
    description = 'Build the extractor lazy loading module'
    user_options = []

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        if self.dry_run:
            print('Skipping build of lazy extractors in dry run mode')
            return
        subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py'])

params = py2exe_params() if sys.argv[1:2] == ['py2exe'] else build_params()
setup(
    name='yt-dlp',
    version=VERSION,
    maintainer='pukkandan',
    maintainer_email='pukkandan.ytdlp@gmail.com',
    description=DESCRIPTION,
    long_description=LONG_DESCRIPTION,
    long_description_content_type='text/markdown',
    url='https://github.com/yt-dlp/yt-dlp',
    packages=packages(),
    install_requires=REQUIREMENTS,
    python_requires='>=3.7',
    project_urls={
        'Documentation': 'https://github.com/yt-dlp/yt-dlp#readme',
        'Source': 'https://github.com/yt-dlp/yt-dlp',
        'Tracker': 'https://github.com/yt-dlp/yt-dlp/issues',
        'Funding': 'https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators',
    },
    classifiers=[
        'Topic :: Multimedia :: Video',
        'Development Status :: 5 - Production/Stable',
        'Environment :: Console',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
        'Programming Language :: Python :: 3.11',
        'Programming Language :: Python :: Implementation',
        'Programming Language :: Python :: Implementation :: CPython',
        'Programming Language :: Python :: Implementation :: PyPy',
        'License :: Public Domain',
        'Operating System :: OS Independent',
    ],
    cmdclass={'build_lazy_extractors': build_lazy_extractors},
    **params
)

if os.getenv('YTDL_PATCHED_INSTALLED_VIA_HOMEBREW') == 'yes':
    # flag this installation as homebrew cellar
    with open('yt_dlp/build_config.py', 'a') as w:
        w.write('''
# Appended by ./setup.py
is_brew = True
''')


code:
import errno
import hashlib
import json
import os.path
import re
import ssl
import sys
import types

import yt_dlp.extractor
from yt_dlp import YoutubeDL
from yt_dlp.compat import compat_os_name
from yt_dlp.utils import preferredencoding, write_string

if 'pytest' in sys.modules:
    import pytest
    is_download_test = pytest.mark.download
else:
    def is_download_test(testClass):
        return testClass

def get_params(override=None):
    PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                   'parameters.json')
    LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                         'local_parameters.json')
    with open(PARAMETERS_FILE, encoding='utf-8') as pf:
        parameters = json.load(pf)
    if os.path.exists(LOCAL_PARAMETERS_FILE):
        with open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf:
            parameters.update(json.load(pf))
    if override:
        parameters.update(override)
    return parameters

def try_rm(filename):
    """ Remove a file if it exists """
    try:
        os.remove(filename)
    except OSError as ose:
        if ose.errno != errno.ENOENT:
            raise

def report_warning(message, *args, **kwargs):
    '''
    Print the message to stderr, it will be prefixed with 'WARNING:'
    If stderr is a tty file the 'WARNING:' will be colored
    '''
    if sys.stderr.isatty() and compat_os_name != 'nt':
        _msg_header = '\033[0;33mWARNING:\033[0m'
    else:
        _msg_header = 'WARNING:'
    output = f'{_msg_header} {message}\n'
    if 'b' in getattr(sys.stderr, 'mode', ''):
        output = output.encode(preferredencoding())
    sys.stderr.write(output)

class FakeYDL(YoutubeDL):
    def __init__(self, override=None):
        # Different instances of the downloader can't share the same dictionary
        # some test set the "sublang" parameter, which would break the md5 checks.
        params = get_params(override=override)
        super().__init__(params, auto_init=False)
        self.result = []

    def to_screen(self, s, *args, **kwargs):
        print(s)

    def trouble(self, s, *args, **kwargs):
        raise Exception(s)

    def download(self, x):
        self.result.append(x)

    def expect_warning(self, regex):
        # Silence an expected warning matching a regex
        old_report_warning = self.report_warning

        def report_warning(self, message, *args, **kwargs):
            if re.match(regex, message):
                return
            old_report_warning(message, *args, **kwargs)
        self.report_warning = types.MethodType(report_warning, self)

def gettestcases(include_onlymatching=False):
    for ie in yt_dlp.extractor.gen_extractors():
        yield from ie.get_testcases(include_onlymatching)

def getwebpagetestcases():
    for ie in yt_dlp.extractor.gen_extractors():
        for tc in ie.get_webpage_testcases():
            tc.setdefault('add_ie', []).append('Generic')
            yield tc

md5 = lambda s: hashlib.md5(s.encode()).hexdigest()

def expect_value(self, got, expected, field):
    if isinstance(expected, str) and expected.startswith('re:'):
        match_str = expected[len('re:'):]
        match_rex = re.compile(match_str)

        self.assertTrue(
            isinstance(got, str),
            f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}')
        self.assertTrue(
            match_rex.match(got),
            f'field {field} (value: {got!r}) should match {match_str!r}')
    elif isinstance(expected, str) and expected.startswith('startswith:'):
        start_str = expected[len('startswith:'):]
        self.assertTrue(
            isinstance(got, str),
            f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}')
        self.assertTrue(
            got.startswith(start_str),
            f'field {field} (value: {got!r}) should start with {start_str!r}')
    elif isinstance(expected, str) and expected.startswith('contains:'):
        contains_str = expected[len('contains:'):]
        self.assertTrue(
            isinstance(got, str),
            f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}')
        self.assertTrue(
            contains_str in got,
            f'field {field} (value: {got!r}) should contain {contains_str!r}')
    elif isinstance(expected, type):
        self.assertTrue(
            isinstance(got, expected),
            f'Expected type {expected!r} for field {field}, but got value {got!r} of type {type(got)!r}')
    elif isinstance(expected, dict) and isinstance(got, dict):
        expect_dict(self, got, expected)
    elif isinstance(expected, list) and isinstance(got, list):
        self.assertEqual(
            len(expected), len(got),
            'Expect a list of length %d, but got a list of length %d for field %s' % (
                len(expected), len(got), field))
        for index, (item_got, item_expected) in enumerate(zip(got, expected)):
            type_got = type(item_got)
            type_expected = type(item_expected)
            self.assertEqual(
                type_expected, type_got,
                'Type mismatch for list item at index %d for field %s, expected %r, got %r' % (
                    index, field, type_expected, type_got))
            expect_value(self, item_got, item_expected, field)
    else:
        if isinstance(expected, str) and expected.startswith('md5:'):
            self.assertTrue(
                isinstance(got, str),
                f'Expected field {field} to be a unicode object, but got value {got!r} of type {type(got)!r}')
            got = 'md5:' + md5(got)
        elif isinstance(expected, str) and re.match(r'^(?:min|max)?count:\d+', expected):
            self.assertTrue(
                isinstance(got, (list, dict)),
                f'Expected field {field} to be a list or a dict, but it is of type {type(got).__name__}')
            op, _, expected_num = expected.partition(':')
            expected_num = int(expected_num)
            if op == 'mincount':
                assert_func = assertGreaterEqual
                msg_tmpl = 'Expected %d items in field %s, but only got %d'
            elif op == 'maxcount':
                assert_func = assertLessEqual
                msg_tmpl = 'Expected maximum %d items in field %s, but got %d'
            elif op == 'count':
                assert_func = assertEqual
                msg_tmpl = 'Expected exactly %d items in field %s, but got %d'
            else:
                assert False
            assert_func(
                self, len(got), expected_num,
                msg_tmpl % (expected_num, field, len(got)))
            return
        self.assertEqual(
            expected, got,
            f'Invalid value for field {field}, expected {expected!r}, got {got!r}')

def expect_dict(self, got_dict, expected_dict):
    for info_field, expected in expected_dict.items():
        got = got_dict.get(info_field)
        expect_value(self, got, expected, info_field)

def sanitize_got_info_dict(got_dict):
    IGNORED_FIELDS = (
        *YoutubeDL._format_fields,

        # Lists
        'formats', 'thumbnails', 'subtitles', 'automatic_captions', 'comments', 'entries',

        # Auto-generated
        'autonumber', 'playlist', 'format_index', 'video_ext', 'audio_ext', 'duration_string', 'epoch',
        'fulltitle', 'extractor', 'extractor_key', 'filepath', 'infojson_filename', 'original_url', 'n_entries',

        # Only live_status needs to be checked
        'is_live', 'was_live',
    )

    IGNORED_PREFIXES = ('', 'playlist', 'requested', 'webpage')

    def sanitize(key, value):
        if isinstance(value, str) and len(value) > 100 and key != 'thumbnail':
            return f'md5:{md5(value)}'
        elif isinstance(value, list) and len(value) > 10:
            return f'count:{len(value)}'
        elif key.endswith('_count') and isinstance(value, int):
            return int
        return value

    test_info_dict = {
        key: sanitize(key, value) for key, value in got_dict.items()
        if value is not None and key not in IGNORED_FIELDS and not any(
            key.startswith(f'{prefix}_') for prefix in IGNORED_PREFIXES)
    }

    # display_id may be generated from id
    if test_info_dict.get('display_id') == test_info_dict.get('id'):
        test_info_dict.pop('display_id')

    return test_info_dict

def expect_info_dict(self, got_dict, expected_dict):
    expect_dict(self, got_dict, expected_dict)
    # Check for the presence of mandatory fields
    if got_dict.get('_type') not in ('playlist', 'multi_video'):
        mandatory_fields = ['id', 'title']
        if expected_dict.get('ext'):
            mandatory_fields.extend(('url', 'ext'))
        for key in mandatory_fields:
            self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key)
    # Check for mandatory fields that are automatically set by YoutubeDL
    for key in ['webpage_url', 'extractor', 'extractor_key']:
        self.assertTrue(got_dict.get(key), 'Missing field: %s' % key)

    test_info_dict = sanitize_got_info_dict(got_dict)

    missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
    if missing_keys:
        def _repr(v):
            if isinstance(v, str):
                return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n')
            elif isinstance(v, type):
                return v.__name__
            else:
                return repr(v)
        info_dict_str = ''
        if len(missing_keys) != len(expected_dict):
            info_dict_str += ''.join(
                f'    {_repr(k)}: {_repr(v)},\n'
                for k, v in test_info_dict.items() if k not in missing_keys)

            if info_dict_str:
                info_dict_str += '\n'
        info_dict_str += ''.join(
            f'    {_repr(k)}: {_repr(test_info_dict[k])},\n'
            for k in missing_keys)
        write_string(
            '\n\'info_dict\': {\n' + info_dict_str + '},\n', out=sys.stderr)
        self.assertFalse(
            missing_keys,
            'Missing keys in test definition: %s' % (
                ', '.join(sorted(missing_keys))))

def assertRegexpMatches(self, text, regexp, msg=None):
    if hasattr(self, 'assertRegexp'):
        return self.assertRegexp(text, regexp, msg)
    else:
        m = re.match(regexp, text)
        if not m:
            note = 'Regexp didn\'t match: %r not found' % (regexp)
            if len(text) < 1000:
                note += ' in %r' % text
            if msg is None:
                msg = note
            else:
                msg = note + ', ' + msg
            self.assertTrue(m, msg)

def assertGreaterEqual(self, got, expected, msg=None):
    if not (got >= expected):
        if msg is None:
            msg = f'{got!r} not greater than or equal to {expected!r}'
        self.assertTrue(got >= expected, msg)

def assertLessEqual(self, got, expected, msg=None):
    if not (got <= expected):
        if msg is None:
            msg = f'{got!r} not less than or equal to {expected!r}'
        self.assertTrue(got <= expected, msg)

def assertEqual(self, got, expected, msg=None):
    if not (got == expected):
        if msg is None:
            msg = f'{got!r} not equal to {expected!r}'
        self.assertTrue(got == expected, msg)

def expect_warnings(ydl, warnings_re):
    real_warning = ydl.report_warning

    def _report_warning(w, *args, **kwargs):
        if not any(re.search(w_re, w) for w_re in warnings_re):
            real_warning(w, *args, **kwargs)

    ydl.report_warning = _report_warning

def http_server_port(httpd):
    if os.name == 'java' and isinstance(httpd.socket, ssl.SSLSocket):
        # In Jython SSLSocket is not a subclass of socket.socket
        sock = httpd.socket.sock
    else:
        sock = httpd.socket
    return sock.getsockname()[1]