etc
/
yt-dlp
mirror of https://github.com/yt-dlp/yt-dlp.git
0
0
Fork 0
Code Issues Projects Releases Wiki Activity

[cleanup] Misc fixes 

Closes #7528
This commit is contained in:
pukkandan 2023-07-22 09:08:12 +05:30
parent e0c4db04dc
commit 62b5c94cad
No known key found for this signature in database
GPG Key ID: 7EEE9E1E817D0A39
12 changed files with 37 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Changelog.md
View File

@ -10,7 +10,7 @@
- Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj)
- `--add-header Cookie:` is deprecated and auto-scoped to input URL domains
- Cookies are scoped when passed to external downloaders
- Add `cookie` field to info.json and deprecate `http_headers.Cookie`
- Add `cookies` field to info.json and deprecate `http_headers.Cookie`
#### Core changes
- [Allow extractors to mark formats as potentially DRM](https://github.com/yt-dlp/yt-dlp/commit/bc344cd456380999c1ee74554dfd432a38f32ec7) ([#7396](https://github.com/yt-dlp/yt-dlp/issues/7396)) by [pukkandan](https://github.com/pukkandan)
@ -51,7 +51,7 @@
- **http**: [Avoid infinite loop when no data is received](https://github.com/yt-dlp/yt-dlp/commit/662ef1e910b72e57957f06589925b2332ba52821) by [pukkandan](https://github.com/pukkandan)
#### Misc. changes
- [Add CodeQL workflow](https://github.com/yt-dlp/yt-dlp/commit/6355b5f1e1e8e7f4ef866d71d51e03baf0e82f17) ([#7497](https://github.com/yt-dlp/yt-dlp/issues/7497)) by [pukkandan](https://github.com/pukkandan)
- [Add CodeQL workflow](https://github.com/yt-dlp/yt-dlp/commit/6355b5f1e1e8e7f4ef866d71d51e03baf0e82f17) ([#7497](https://github.com/yt-dlp/yt-dlp/issues/7497)) by [jorgectf](https://github.com/jorgectf)
- **cleanup**: Miscellaneous: [337734d](https://github.com/yt-dlp/yt-dlp/commit/337734d4a8a6500bc65434843db346b5cbd05e81) by [pukkandan](https://github.com/pukkandan)
- **docs**: [Minor fixes](https://github.com/yt-dlp/yt-dlp/commit/b532a3481046e1eabb6232ee8196fb696c356ff6) by [pukkandan](https://github.com/pukkandan)
- **make_changelog**: [Skip reverted commits](https://github.com/yt-dlp/yt-dlp/commit/fa44802809d189fca0f4782263d48d6533384503) by [pukkandan](https://github.com/pukkandan)

4
README.md
View File

@ -1569,7 +1569,7 @@ The available fields are:
- `aext`: Audio Extension (`m4a` > `aac` > `mp3` > `ogg` > `opus` > `webm` > other). If `--prefer-free-formats` is used, the order changes to `ogg` > `opus` > `webm` > `mp3` > `m4a` > `aac`
- `ext`: Equivalent to `vext,aext`
- `filesize`: Exact filesize, if known in advance
- `fs_approx`: Approximate filesize calculated from the manifests
- `fs_approx`: Approximate filesize
- `size`: Exact filesize if available, otherwise approximate filesize
- `height`: Height of video
- `width`: Width of video
@ -1580,7 +1580,7 @@ The available fields are:
- `tbr`: Total average bitrate in KBit/s
- `vbr`: Average video bitrate in KBit/s
- `abr`: Average audio bitrate in KBit/s
- `br`: Equivalent to using `tbr,vbr,abr`
- `br`: Average bitrate in KBit/s, `tbr`/`vbr`/`abr`
- `asr`: Audio sample rate in Hz
**Deprecation warning**: Many of these fields have (currently undocumented) aliases, that may be removed in a future version. It is recommended to use only the documented field names.

7
devscripts/changelog_override.json
View File

@ -63,6 +63,11 @@
{
"action": "add",
"when": "1ceb657bdd254ad961489e5060f2ccc7d556b729",
"short": "[priority] Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj)\n - `--add-header Cookie:` is deprecated and auto-scoped to input URL domains\n - Cookies are scoped when passed to external downloaders\n - Add `cookie` field to info.json and deprecate `http_headers.Cookie`"
"short": "[priority] Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj)\n - `--add-header Cookie:` is deprecated and auto-scoped to input URL domains\n - Cookies are scoped when passed to external downloaders\n - Add `cookies` field to info.json and deprecate `http_headers.Cookie`"
},
{
"action": "change",
"when": "b03fa7834579a01cc5fba48c0e73488a16683d48",
"short": "[ie/twitter] Revert 92315c03774cfabb3a921884326beb4b981f786b"
}
]

4
devscripts/make_changelog.py
View File

@ -53,10 +53,10 @@ class CommitGroup(enum.Enum):
'cookies',
'core',
'dependencies',
'formats',
'jsinterp',
'networking',
'outtmpl',
'formats',
'plugins',
'update',
'upstream',
@ -254,7 +254,7 @@ class CommitRange:
(?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
''', re.VERBOSE | re.DOTALL)
EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE)
REVERT_RE = re.compile(r'(?i:Revert)\s+([\da-f]{40})')
REVERT_RE = re.compile(r'(?:\[[^\]]+\]\s+)?(?i:Revert)\s+([\da-f]{40})')
FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})')
UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)')

3
test/test_YoutubeDL.py
View File

@ -26,7 +26,6 @@ from yt_dlp.utils import (
)
from yt_dlp.utils.traversal import traverse_obj
TEST_URL = 'http://localhost/sample.mp4'
@ -687,7 +686,7 @@ class TestYoutubeDL(unittest.TestCase):
test('%(duration_string)s', ('27:46:40', '27-46-40'))
test('%(resolution)s', '1080p')
test('%(playlist_index|)s', '001')
test('%(playlist_index&{}!)s', '001!')
test('%(playlist_index&{}!)s', '1!')
test('%(playlist_autonumber)s', '02')
test('%(autonumber)s', '00001')
test('%(autonumber+2)03d', '005', autonumber_start=3)

12
test/test_YoutubeDLCookieJar.py
View File

@ -17,10 +17,10 @@ from yt_dlp.cookies import YoutubeDLCookieJar
class TestYoutubeDLCookieJar(unittest.TestCase):
def test_keep_session_cookies(self):
cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt')
cookiejar.load(ignore_discard=True, ignore_expires=True)
cookiejar.load()
tf = tempfile.NamedTemporaryFile(delete=False)
try:
cookiejar.save(filename=tf.name, ignore_discard=True, ignore_expires=True)
cookiejar.save(filename=tf.name)
temp = tf.read().decode()
self.assertTrue(re.search(
r'www\.foobar\.foobar\s+FALSE\s+/\s+TRUE\s+0\s+YoutubeDLExpiresEmpty\s+YoutubeDLExpiresEmptyValue', temp))
@ -32,7 +32,7 @@ class TestYoutubeDLCookieJar(unittest.TestCase):
def test_strip_httponly_prefix(self):
cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt')
cookiejar.load(ignore_discard=True, ignore_expires=True)
cookiejar.load()
def assert_cookie_has_value(key):
self.assertEqual(cookiejar._cookies['www.foobar.foobar']['/'][key].value, key + '_VALUE')
@ -42,20 +42,20 @@ class TestYoutubeDLCookieJar(unittest.TestCase):
def test_malformed_cookies(self):
cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/malformed_cookies.txt')
cookiejar.load(ignore_discard=True, ignore_expires=True)
cookiejar.load()
# Cookies should be empty since all malformed cookie file entries
# will be ignored
self.assertFalse(cookiejar._cookies)
def test_get_cookie_header(self):
cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt')
cookiejar.load(ignore_discard=True, ignore_expires=True)
cookiejar.load()
header = cookiejar.get_cookie_header('https://www.foobar.foobar')
self.assertIn('HTTPONLY_COOKIE', header)
def test_get_cookies_for_url(self):
cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt')
cookiejar.load(ignore_discard=True, ignore_expires=True)
cookiejar.load()
cookies = cookiejar.get_cookies_for_url('https://www.foobar.foobar/')
self.assertEqual(len(cookies), 2)
cookies = cookiejar.get_cookies_for_url('https://foobar.foobar/')

13
yt_dlp/YoutubeDL.py
View File

@ -572,7 +572,7 @@ class YoutubeDL:
'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels',
'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns',
'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start',
'preference', 'language', 'language_preference', 'quality', 'source_preference',
'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies',
'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'hls_aes', 'downloader_options',
'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time'
}
@ -621,7 +621,8 @@ class YoutubeDL:
if self.params.get('no_color'):
if self.params.get('color') is not None:
self.report_warning('Overwriting params from "color" with "no_color"')
self.params.setdefault('_warnings', []).append(
'Overwriting params from "color" with "no_color"')
self.params['color'] = 'no_color'
term_allow_color = os.environ.get('TERM', '').lower() != 'dumb'
@ -949,7 +950,7 @@ class YoutubeDL:
def save_cookies(self):
if self.params.get('cookiefile') is not None:
self.cookiejar.save(ignore_discard=True, ignore_expires=True)
self.cookiejar.save()
def __exit__(self, *args):
self.restore_console_title()
@ -3290,7 +3291,7 @@ class YoutubeDL:
fd, success = None, True
if info_dict.get('protocol') or info_dict.get('url'):
fd = get_suitable_downloader(info_dict, self.params, to_stdout=temp_filename == '-')
if fd is not FFmpegFD and 'no-direct-merge' not in self.params['compat_opts'] and (
if fd != FFmpegFD and 'no-direct-merge' not in self.params['compat_opts'] and (
info_dict.get('section_start') or info_dict.get('section_end')):
msg = ('This format cannot be partially downloaded' if FFmpegFD.available()
else 'You have requested downloading the video partially, but ffmpeg is not installed')
@ -3451,7 +3452,7 @@ class YoutubeDL:
postprocessed_by_ffmpeg = info_dict.get('requested_formats') or any((
isinstance(pp, FFmpegVideoConvertorPP)
and resolve_recode_mapping(ext, pp.mapping)[0] not in (ext, None)
) for pp in self._pps['post_process'])
) for pp in self._pps['post_process']) or fd == FFmpegFD
if not postprocessed_by_ffmpeg:
ffmpeg_fixup(ext == 'm4a' and info_dict.get('container') == 'm4a_dash',
@ -4031,7 +4032,7 @@ class YoutubeDL:
"""
Get a urllib OpenerDirector from the Urllib handler (deprecated).
"""
self.deprecation_warning('YoutubeDL._opener() is deprecated, use YoutubeDL.urlopen()')
self.deprecation_warning('YoutubeDL._opener is deprecated, use YoutubeDL.urlopen()')
handler = self._request_director.handlers['Urllib']
return handler._get_instance(cookiejar=self.cookiejar, proxies=self.proxies)

4
yt_dlp/compat/_legacy.py
View File

@ -16,12 +16,12 @@ import shlex
import shutil
import socket
import struct
import subprocess
import tokenize
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as etree
from subprocess import DEVNULL
# isort: split
import asyncio # noqa: F401
@ -85,7 +85,7 @@ compat_socket_create_connection = socket.create_connection
compat_Struct = struct.Struct
compat_struct_pack = struct.pack
compat_struct_unpack = struct.unpack
compat_subprocess_get_DEVNULL = lambda: DEVNULL
compat_subprocess_get_DEVNULL = lambda: subprocess.DEVNULL
compat_tokenize_tokenize = tokenize.tokenize
compat_urllib_error = urllib.error
compat_urllib_HTTPError = urllib.error.HTTPError

10
yt_dlp/cookies.py
View File

@ -97,7 +97,7 @@ def load_cookies(cookie_file, browser_specification, ydl):
jar = YoutubeDLCookieJar(cookie_file)
if not is_filename or os.access(cookie_file, os.R_OK):
jar.load(ignore_discard=True, ignore_expires=True)
jar.load()
cookie_jars.append(jar)
return _merge_cookie_jars(cookie_jars)
@ -1213,7 +1213,7 @@ class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
file.truncate(0)
yield file
def _really_save(self, f, ignore_discard=False, ignore_expires=False):
def _really_save(self, f, ignore_discard, ignore_expires):
now = time.time()
for cookie in self:
if (not ignore_discard and cookie.discard
@ -1234,7 +1234,7 @@ class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
name, value
)))
def save(self, filename=None, *args, **kwargs):
def save(self, filename=None, ignore_discard=True, ignore_expires=True):
"""
Save cookies to a file.
Code is taken from CPython 3.6
@ -1253,9 +1253,9 @@ class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
with self.open(filename, write=True) as f:
f.write(self._HEADER)
self._really_save(f, *args, **kwargs)
self._really_save(f, ignore_discard, ignore_expires)
def load(self, filename=None, ignore_discard=False, ignore_expires=False):
def load(self, filename=None, ignore_discard=True, ignore_expires=True):
"""Load cookies from a file."""
if filename is None:
if self.filename is not None:

2
yt_dlp/downloader/external.py
View File

@ -137,7 +137,7 @@ class ExternalFD(FragmentFD):
self._cookies_tempfile = tmp_cookies.name
self.to_screen(f'[download] Writing temporary cookies file to "{self._cookies_tempfile}"')
# real_download resets _cookies_tempfile; if it's None then save() will write to cookiejar.filename
self.ydl.cookiejar.save(self._cookies_tempfile, ignore_discard=True, ignore_expires=True)
self.ydl.cookiejar.save(self._cookies_tempfile)
return self.ydl.cookiejar.filename or self._cookies_tempfile
def _call_downloader(self, tmpfilename, info_dict):

4
yt_dlp/networking/_urllib.py
View File

@ -28,7 +28,7 @@ from ._helper import (
make_socks_proxy_opts,
select_proxy,
)
from .common import Features, RequestHandler, Response, register
from .common import Features, RequestHandler, Response, register_rh
from .exceptions import (
CertificateVerifyError,
HTTPError,
@ -372,7 +372,7 @@ def handle_response_read_exceptions(e):
raise TransportError(cause=e) from e
@register
@register_rh
class UrllibRH(RequestHandler, InstanceStoreMixin):
_SUPPORTED_URL_SCHEMES = ('http', 'https', 'data', 'ftp')
_SUPPORTED_PROXY_SCHEMES = ('http', 'socks4', 'socks4a', 'socks5', 'socks5h')

2
yt_dlp/networking/common.py
View File

@ -105,7 +105,7 @@ class RequestDirector:
_REQUEST_HANDLERS = {}
def register(handler):
def register_rh(handler):
"""Register a RequestHandler class"""
assert issubclass(handler, RequestHandler), f'{handler} must be a subclass of RequestHandler'
assert handler.RH_KEY not in _REQUEST_HANDLERS, f'RequestHandler {handler.RH_KEY} already registered'