[cleanup] Misc cleanup

Closes #1805, closes #1800

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -34,7 +34,7 @@ body:
       label: Example URLs
       description: |
         Provide all kinds of example URLs for which support should be added
-      value: |
+      placeholder: |
         - Single video: https://www.youtube.com/watch?v=BaW_jenozKc
         - Single video: https://youtu.be/BaW_jenozKc
         - Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc

.gitignore

@@ -6,41 +6,47 @@ cookies
 .netrc
 
 # Downloaded
-*.srt
-*.ttml
-*.sbv
-*.vtt
-*.flv
-*.mp4
-*.m4a
-*.m4v
-*.mp3
 *.3gp
-*.webm
+*.annotations.xml
-*.wav
 *.ape
-*.mkv
+*.aria2
-*.flac
 *.avi
-*.swf
+*.description
-*.part
+*.desktop
-*.part-*
-*.ytdl
 *.dump
+*.flac
+*.flv
 *.frag
 *.frag.urls
-*.aria2
+*.info.json
-*.swp
+*.jpeg
+*.jpg
+*.live_chat.json
+*.m4a
+*.m4v
+*.mhtml
+*.mkv
+*.mov
+*.mp3
+*.mp4
 *.ogg
 *.opus
-*.info.json
+*.part
-*.live_chat.json
+*.part-*
-*.jpg
-*.jpeg
 *.png
+*.sbv
+*.srt
+*.swf
+*.swp
+*.ttml
+*.unknown_video
+*.url
+*.vtt
+*.wav
+*.webloc
+*.webm
 *.webp
-*.annotations.xml
+*.ytdl
-*.description
 .cache/
 
 # Allow config/media files in testdata

Makefile

@@ -15,9 +15,11 @@ pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites com
 clean-test:
 	rm -rf *.3gp *.annotations.xml *.ape *.avi *.description *.dump *.flac *.flv *.frag *.frag.aria2 *.frag.urls \
 	*.info.json *.jpeg *.jpg *.live_chat.json *.m4a *.m4v *.mkv *.mp3 *.mp4 *.ogg *.opus *.part* *.png *.sbv *.srt \
-	*.swf *.swp *.ttml *.vtt *.wav *.webm *.webp *.ytdl test/testdata/player-*.js
+	*.swf *.swp *.ttml *.vtt *.wav *.webm *.webp *.mhtml *.mov *.unknown_video *.desktop *.url *.webloc *.ytdl \
+	test/testdata/player-*.js tmp/
 clean-dist:
-	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap
+	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
+	yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap
 clean-cache:
 	find . -name "*.pyc" -o -name "*.class" -delete
 
@@ -31,7 +33,6 @@ DESTDIR ?= .
 BINDIR ?= $(PREFIX)/bin
 MANDIR ?= $(PREFIX)/man
 SHAREDIR ?= $(PREFIX)/share
-# make_supportedsites.py doesnot work correctly in python2
 PYTHON ?= /usr/bin/env python3
 
 # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local

README.md

@@ -1204,7 +1204,7 @@ # OUTPUT TEMPLATE
 
 The current default template is `%(title)s [%(id)s].%(ext)s`.
 
-In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
+In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title.
 
 #### Output template and Windows batch files
 
@@ -1614,7 +1614,7 @@ # EMBEDDING YT-DLP
     ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L154-L452).
+Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L162).
 
 Here's a more complete example demonstrating various functionality:
 
@@ -1785,7 +1785,7 @@ #### Old aliases
     --yes-overwrites                 --force-overwrites
 
 #### Sponskrub Options
-Support for [SponSkrub](https://github.com/faissaloo/SponSkrub) has been deprecated in favor of `--sponsorblock`
+Support for [SponSkrub](https://github.com/faissaloo/SponSkrub) has been deprecated in favor of the `--sponsorblock` options
 
     --sponskrub                      --sponsorblock-mark all
     --no-sponskrub                   --no-sponsorblock

yt_dlp/YoutubeDL.py

@@ -848,7 +848,7 @@ def to_screen(self, message, skip_eol=False):
 
     class Styles(Enum):
         HEADERS = 'yellow'
-        EMPHASIS = 'blue'
+        EMPHASIS = 'light blue'
         ID = 'green'
         DELIM = 'blue'
         ERROR = 'red'
@@ -863,7 +863,7 @@ def _format_text(self, handle, allow_colors, text, f, fallback=None, *, test_enc
             if fallback is not None and text != original_text:
                 text = fallback
         if isinstance(f, self.Styles):
-            f = f._value_
+            f = f.value
         return format_text(text, f) if allow_colors else text if fallback is None else fallback
 
     def _format_screen(self, *args, **kwargs):
@@ -3223,15 +3223,19 @@ def format_resolution(format, default='unknown'):
     def _format_note(self, fdict):
         res = ''
         if fdict.get('ext') in ['f4f', 'f4m']:
-            res += '(unsupported) '
+            res += '(unsupported)'
         if fdict.get('language'):
             if res:
                 res += ' '
-            res += '[%s] ' % fdict['language']
+            res += '[%s]' % fdict['language']
         if fdict.get('format_note') is not None:
-            res += fdict['format_note'] + ' '
+            if res:
+                res += ' '
+            res += fdict['format_note']
         if fdict.get('tbr') is not None:
-            res += '%4dk ' % fdict['tbr']
+            if res:
+                res += ', '
+            res += '%4dk' % fdict['tbr']
         if fdict.get('container') is not None:
             if res:
                 res += ', '

yt_dlp/__init__.py

@@ -25,18 +25,16 @@
 from .utils import (
     DateRange,
     decodeOption,
+    DownloadCancelled,
     DownloadError,
     error_to_compat_str,
-    ExistingVideoReached,
     expand_path,
     float_or_none,
     int_or_none,
     match_filter_func,
-    MaxDownloadsReached,
     parse_duration,
     preferredencoding,
     read_batch_urls,
-    RejectedVideoReached,
     render_table,
     SameFileError,
     setproctitle,
@@ -195,7 +193,7 @@ def _real_main(argv=None):
     if opts.overwrites:  # --yes-overwrites implies --no-continue
         opts.continue_dl = False
     if opts.concurrent_fragment_downloads <= 0:
-        raise ValueError('Concurrent fragments must be positive')
+        parser.error('Concurrent fragments must be positive')
     if opts.wait_for_video is not None:
         mobj = re.match(r'(?P<min>\d+)(?:-(?P<max>\d+))?$', opts.wait_for_video)
         if not mobj:
@@ -231,9 +229,9 @@ def parse_retries(retries, name=''):
             parser.error('invalid http chunk size specified')
         opts.http_chunk_size = numeric_chunksize
     if opts.playliststart <= 0:
-        raise ValueError('Playlist start must be positive')
+        raise parser.error('Playlist start must be positive')
     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
-        raise ValueError('Playlist end must be greater than playlist start')
+        raise parser.error('Playlist end must be greater than playlist start')
     if opts.extractaudio:
         opts.audioformat = opts.audioformat.lower()
         if opts.audioformat not in ['best'] + list(FFmpegExtractAudioPP.SUPPORTED_EXTS):
@@ -762,7 +760,7 @@ def report_args_compat(arg, name):
     }
 
     with YoutubeDL(ydl_opts) as ydl:
-        actual_use = len(all_urls) or opts.load_info_filename
+        actual_use = all_urls or opts.load_info_filename
 
         # Remove cache dir
         if opts.rm_cachedir:
@@ -791,7 +789,7 @@ def report_args_compat(arg, name):
                 retcode = ydl.download_with_info_file(expand_path(opts.load_info_filename))
             else:
                 retcode = ydl.download(all_urls)
-        except (MaxDownloadsReached, ExistingVideoReached, RejectedVideoReached):
+        except DownloadCancelled:
             ydl.to_screen('Aborting remaining downloads')
             retcode = 101
 

yt_dlp/downloader/__init__.py

@@ -41,6 +41,7 @@ def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=N
 
 PROTOCOL_MAP = {
     'rtmp': RtmpFD,
+    'rtmpe': RtmpFD,
     'rtmp_ffmpeg': FFmpegFD,
     'm3u8_native': HlsFD,
     'm3u8': FFmpegFD,

yt_dlp/downloader/common.py

@@ -93,6 +93,8 @@ def calc_percent(byte_counter, data_len):
     def format_percent(percent):
         if percent is None:
             return '---.-%'
+        elif percent == 100:
+            return '100%'
         return '%6s' % ('%3.1f%%' % percent)
 
     @staticmethod
@@ -298,7 +300,7 @@ def report_progress(self, s):
                 s['_elapsed_str'] = self.format_seconds(s['elapsed'])
                 msg_template += ' in %(_elapsed_str)s'
             s['_percent_str'] = self.format_percent(100)
-            self._report_progress_status(s)
+            self._report_progress_status(s, msg_template)
             return
 
         if s['status'] != 'downloading':
@@ -307,7 +309,7 @@ def report_progress(self, s):
         if s.get('eta') is not None:
             s['_eta_str'] = self.format_eta(s['eta'])
         else:
-            s['_eta_str'] = 'Unknown ETA'
+            s['_eta_str'] = 'Unknown'
 
         if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
@@ -339,7 +341,7 @@ def report_progress(self, s):
                 else:
                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
             else:
-                msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
+                msg_template = '%(_percent_str)s at %(_speed_str)s ETA %(_eta_str)s'
         if s.get('fragment_index') and s.get('fragment_count'):
             msg_template += ' (frag %(fragment_index)s/%(fragment_count)s)'
         elif s.get('fragment_index'):

yt_dlp/extractor/bbc.py

@@ -472,8 +472,7 @@ def _download_playlist(self, playlist_id):
                             f['language_preference'] = -10
                     formats += version_formats
                     for tag, subformats in (version_subtitles or {}).items():
-                        subtitles.setdefault(tag, [])
+                        subtitles.setdefault(tag, []).extend(subformats)
-                        subtitles[tag] += subformats
 
             return programme_id, title, description, duration, formats, subtitles
         except ExtractorError as ee:

yt_dlp/extractor/common.py

@@ -1538,10 +1538,10 @@ class FormatSort:
 
         default = ('hidden', 'aud_or_vid', 'hasvid', 'ie_pref', 'lang', 'quality',
                    'res', 'fps', 'hdr:12', 'codec:vp9.2', 'size', 'br', 'asr',
-                   'proto', 'ext', 'hasaud', 'source', 'format_id')  # These must not be aliases
+                   'proto', 'ext', 'hasaud', 'source', 'id')  # These must not be aliases
         ytdl_default = ('hasaud', 'lang', 'quality', 'tbr', 'filesize', 'vbr',
                         'height', 'width', 'proto', 'vext', 'abr', 'aext',
-                        'fps', 'fs_approx', 'source', 'format_id')
+                        'fps', 'fs_approx', 'source', 'id')
 
         settings = {
             'vcodec': {'type': 'ordered', 'regex': True,
@@ -1551,7 +1551,7 @@ class FormatSort:
             'hdr': {'type': 'ordered', 'regex': True, 'field': 'dynamic_range',
                     'order': ['dv', '(hdr)?12', r'(hdr)?10\+', '(hdr)?10', 'hlg', '', 'sdr', None]},
             'proto': {'type': 'ordered', 'regex': True, 'field': 'protocol',
-                      'order': ['(ht|f)tps', '(ht|f)tp$', 'm3u8.+', '.*dash', 'ws|websocket', '', 'mms|rtsp', 'none', 'f4']},
+                      'order': ['(ht|f)tps', '(ht|f)tp$', 'm3u8.*', '.*dash', 'websocket_frag', 'rtmpe?', '', 'mms|rtsp', 'ws|websocket', 'f4']},
             'vext': {'type': 'ordered', 'field': 'video_ext',
                      'order': ('mp4', 'webm', 'flv', '', 'none'),
                      'order_free': ('webm', 'mp4', 'flv', '', 'none')},
@@ -1615,7 +1615,12 @@ class FormatSort:
             'format_id': {'type': 'alias', 'field': 'id'},
         }
 
-        _order = []
+        def __init__(self, ie, field_preference):
+            self._order = []
+            self.ydl = ie._downloader
+            self.evaluate_params(self.ydl.params, field_preference)
+            if ie.get_param('verbose'):
+                self.print_verbose_info(self.ydl.write_debug)
 
         def _get_field_setting(self, field, key):
             if field not in self.settings:
@@ -1805,10 +1810,7 @@ def calculate_preference(self, format):
     def _sort_formats(self, formats, field_preference=[]):
         if not formats:
             return
-        format_sort = self.FormatSort()  # params and to_screen are taken from the downloader
+        format_sort = self.FormatSort(self, field_preference)
-        format_sort.evaluate_params(self._downloader.params, field_preference)
-        if self.get_param('verbose', False):
-            format_sort.print_verbose_info(self._downloader.write_debug)
         formats.sort(key=lambda f: format_sort.calculate_preference(f))
 
     def _check_formats(self, formats, video_id):

yt_dlp/extractor/instagram.py

@@ -499,7 +499,7 @@ def _query_vars_for(data):
 
 class InstagramTagIE(InstagramPlaylistBaseIE):
     _VALID_URL = r'https?://(?:www\.)?instagram\.com/explore/tags/(?P<id>[^/]+)'
-    IE_DESC = 'Instagram hashtag search'
+    IE_DESC = 'Instagram hashtag search URLs'
     IE_NAME = 'instagram:tag'
     _TESTS = [{
         'url': 'https://instagram.com/explore/tags/lolcats',

yt_dlp/extractor/mlssoccer.py

@@ -21,7 +21,6 @@ class MLSSoccerIE(InfoExtractor):
             'uploader_id': '5530036772001',
             'tags': ['club/canada'],
             'is_live': False,
-            'duration_string': '5:50',
             'upload_date': '20211007',
             'filesize_approx': 255193528.83200002
         },

yt_dlp/extractor/niconico.py

@@ -703,7 +703,7 @@ def _real_extract(self, url):
 
 
 class NicovideoSearchIE(SearchInfoExtractor, NicovideoSearchURLIE):
-    IE_DESC = 'Nico video searches'
+    IE_DESC = 'Nico video search'
     IE_NAME = NicovideoSearchIE_NAME
     _SEARCH_KEY = 'nicosearch'
     _TESTS = []
@@ -714,7 +714,7 @@ def _search_results(self, query):
 
 
 class NicovideoSearchDateIE(NicovideoSearchIE):
-    IE_DESC = 'Nico video searches, newest first'
+    IE_DESC = 'Nico video search, newest first'
     IE_NAME = f'{NicovideoSearchIE_NAME}:date'
     _SEARCH_KEY = 'nicosearchdate'
     _TESTS = [{

yt_dlp/options.py

@@ -377,10 +377,6 @@ def _dict_from_options_callback(
         '--reject-title',
         dest='rejecttitle', metavar='REGEX',
         help=optparse.SUPPRESS_HELP)
-    selection.add_option(
-        '--max-downloads',
-        dest='max_downloads', metavar='NUMBER', type=int, default=None,
-        help='Abort after downloading NUMBER files')
     selection.add_option(
         '--min-filesize',
         metavar='SIZE', dest='min_filesize', default=None,
@@ -451,6 +447,14 @@ def _dict_from_options_callback(
         '--download-archive', metavar='FILE',
         dest='download_archive',
         help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it')
+    selection.add_option(
+        '--no-download-archive',
+        dest='download_archive', action="store_const", const=None,
+        help='Do not use archive file (default)')
+    selection.add_option(
+        '--max-downloads',
+        dest='max_downloads', metavar='NUMBER', type=int, default=None,
+        help='Abort after downloading NUMBER files')
     selection.add_option(
         '--break-on-existing',
         action='store_true', dest='break_on_existing', default=False,
@@ -471,10 +475,6 @@ def _dict_from_options_callback(
         '--skip-playlist-after-errors', metavar='N',
         dest='skip_playlist_after_errors', default=None, type=int,
         help='Number of allowed failures until the rest of the playlist is skipped')
-    selection.add_option(
-        '--no-download-archive',
-        dest='download_archive', action="store_const", const=None,
-        help='Do not use archive file (default)')
     selection.add_option(
         '--include-ads',
         dest='include_ads', action='store_true',
@@ -1154,7 +1154,7 @@ def _dict_from_options_callback(
     filesystem.add_option(
         '--cookies',
         dest='cookiefile', metavar='FILE',
-        help='File to read cookies from and dump cookie jar in')
+        help='Netscape formatted file to read cookies from and dump cookie jar in')
     filesystem.add_option(
         '--no-cookies',
         action='store_const', const=None, dest='cookiefile', metavar='FILE',
@@ -1354,7 +1354,7 @@ def _dict_from_options_callback(
             'Automatically correct known faults of the file. '
             'One of never (do nothing), warn (only emit a warning), '
             'detect_or_warn (the default; fix file if we can, warn otherwise), '
-            'force (try fixing even if file already exists'))
+            'force (try fixing even if file already exists)'))
     postproc.add_option(
         '--prefer-avconv', '--no-prefer-ffmpeg',
         action='store_false', dest='prefer_ffmpeg',

yt_dlp/postprocessor/metadataparser.py

@@ -16,7 +16,7 @@ def __init__(self, downloader, actions):
         for f in actions:
             action = f[0]
             assert isinstance(action, self.Actions)
-            self._actions.append(getattr(self, action._value_)(*f[1:]))
+            self._actions.append(getattr(self, action.value)(*f[1:]))
 
     @classmethod
     def validate_action(cls, action, *data):
@@ -26,7 +26,7 @@ def validate_action(cls, action, *data):
         '''
         if not isinstance(action, cls.Actions):
             raise ValueError(f'{action!r} is not a valid action')
-        getattr(cls, action._value_)(cls, *data)
+        getattr(cls, action.value)(cls, *data)
 
     @staticmethod
     def field_to_template(tmpl):

yt_dlp/utils.py

@@ -2575,10 +2575,6 @@ class PostProcessingError(YoutubeDLError):
     indicate an error in the postprocessing task.
     """
 
-    def __init__(self, msg):
-        super(PostProcessingError, self).__init__(msg)
-        self.msg = msg
-
 
 class DownloadCancelled(YoutubeDLError):
     """ Exception raised when the download queue should be interrupted """