[core] Deprecate internal Youtubedl-no-compression header (#6876)

Authored by: coletdjnz

yt_dlp/YoutubeDL.py

@@ -2380,7 +2380,9 @@ def restore_last_token(self):
 
     def _calc_headers(self, info_dict):
         res = merge_headers(self.params['http_headers'], info_dict.get('http_headers') or {})
-
+        if 'Youtubedl-No-Compression' in res:  # deprecated
+            res.pop('Youtubedl-No-Compression', None)
+            res['Accept-Encoding'] = 'identity'
         cookies = self._calc_cookies(info_dict['url'])
         if cookies:
             res['Cookie'] = cookies

yt_dlp/downloader/external.py

@@ -23,7 +23,6 @@
     encodeArgument,
     encodeFilename,
     find_available_port,
-    handle_youtubedl_headers,
     remove_end,
     sanitized_Request,
     traverse_obj,
@@ -529,10 +528,9 @@ def _call_downloader(self, tmpfilename, info_dict):
         selected_formats = info_dict.get('requested_formats') or [info_dict]
         for i, fmt in enumerate(selected_formats):
             if fmt.get('http_headers') and re.match(r'^https?://', fmt['url']):
-                headers_dict = handle_youtubedl_headers(fmt['http_headers'])
                 # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
                 # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
-                args.extend(['-headers', ''.join(f'{key}: {val}\r\n' for key, val in headers_dict.items())])
+                args.extend(['-headers', ''.join(f'{key}: {val}\r\n' for key, val in fmt['http_headers'].items())])
 
             if start_time:
                 args += ['-ss', str(start_time)]

yt_dlp/downloader/http.py

@@ -45,8 +45,8 @@ class DownloadContext(dict):
         ctx.tmpfilename = self.temp_name(filename)
         ctx.stream = None
 
-        # Do not include the Accept-Encoding header
+        # Disable compression
-        headers = {'Youtubedl-no-compression': 'True'}
+        headers = {'Accept-Encoding': 'identity'}
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)

yt_dlp/extractor/litv.py

@@ -113,7 +113,7 @@ def _real_extract(self, url):
             entry_protocol='m3u8_native', m3u8_id='hls')
         for a_format in formats:
             # LiTV HLS segments doesn't like compressions
-            a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = True
+            a_format.setdefault('http_headers', {})['Accept-Encoding'] = 'identity'
 
         title = program_info['title'] + program_info.get('secondaryMark', '')
         description = program_info.get('description')

yt_dlp/utils/_legacy.py

@@ -161,3 +161,13 @@ def register_socks_protocols():
     for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
         if scheme not in urllib.parse.uses_netloc:
             urllib.parse.uses_netloc.append(scheme)
+
+
+def handle_youtubedl_headers(headers):
+    filtered_headers = headers
+
+    if 'Youtubedl-no-compression' in filtered_headers:
+        filtered_headers = {k: v for k, v in filtered_headers.items() if k.lower() != 'accept-encoding'}
+        del filtered_headers['Youtubedl-no-compression']
+
+    return filtered_headers

yt_dlp/utils/_utils.py

@@ -1308,25 +1308,12 @@ def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_a
     return hc
 
 
-def handle_youtubedl_headers(headers):
-    filtered_headers = headers
-
-    if 'Youtubedl-no-compression' in filtered_headers:
-        filtered_headers = {k: v for k, v in filtered_headers.items() if k.lower() != 'accept-encoding'}
-        del filtered_headers['Youtubedl-no-compression']
-
-    return filtered_headers
-
-
 class YoutubeDLHandler(urllib.request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
     This class, when installed with an OpenerDirector, automatically adds
-    the standard headers to every HTTP request and handles gzipped and
+    the standard headers to every HTTP request and handles gzipped, deflated and
-    deflated responses from web servers. If compression is to be avoided in
+    brotli responses from web servers.
-    a particular request, the original request in the program code only has
-    to include the HTTP header "Youtubedl-no-compression", which will be
-    removed before making the real request.
 
     Part of this code was copied from:
 
@@ -1389,11 +1376,13 @@ def http_request(self, req):
             if h.capitalize() not in req.headers:
                 req.add_header(h, v)
 
+        if 'Youtubedl-no-compression' in req.headers:  # deprecated
+            req.headers.pop('Youtubedl-no-compression', None)
+            req.add_header('Accept-encoding', 'identity')
+
         if 'Accept-encoding' not in req.headers:
             req.add_header('Accept-encoding', ', '.join(SUPPORTED_ENCODINGS))
 
-        req.headers = handle_youtubedl_headers(req.headers)
-
         return super().do_request_(req)
 
     def http_response(self, req, resp):