[extractor/bravotv] Fix extractor (#6568)

Closes #6562
Authored by: bashonly

yt_dlp/extractor/bravotv.py

@@ -1,117 +1,185 @@
-import re
-
 from .adobepass import AdobePassIE
 from ..utils import (
-    smuggle_url,
+    extract_attributes,
-    update_url_query,
-    int_or_none,
     float_or_none,
-    try_get,
+    get_element_html_by_class,
-    dict_get,
+    int_or_none,
+    merge_dicts,
+    parse_age_limit,
+    remove_end,
+    str_or_none,
+    traverse_obj,
+    unescapeHTML,
+    unified_timestamp,
+    update_url_query,
+    url_or_none,
 )
 
 
 class BravoTVIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?(?P<req_id>bravotv|oxygen)\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>bravotv|oxygen)\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.bravotv.com/top-chef/season-16/episode-15/videos/the-top-chef-season-16-winner-is',
-        'md5': 'e34684cfea2a96cd2ee1ef3a60909de9',
         'info_dict': {
-            'id': 'epL0pmK1kQlT',
+            'id': '3923059',
             'ext': 'mp4',
             'title': 'The Top Chef Season 16 Winner Is...',
             'description': 'Find out who takes the title of Top Chef!',
-            'uploader': 'NBCU-BRAV',
             'upload_date': '20190314',
             'timestamp': 1552591860,
             'season_number': 16,
             'episode_number': 15,
             'series': 'Top Chef',
             'episode': 'The Top Chef Season 16 Winner Is...',
-            'duration': 190.0,
+            'duration': 190.357,
-        }
+            'season': 'Season 16',
+            'thumbnail': r're:^https://.+\.jpg',
+        },
+        'params': {'skip_download': 'm3u8'},
     }, {
-        'url': 'http://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
+        'url': 'https://www.bravotv.com/top-chef/season-20/episode-1/london-calling',
-        'only_matching': True,
+        'info_dict': {
+            'id': '9000234570',
+            'ext': 'mp4',
+            'title': 'London Calling',
+            'description': 'md5:5af95a8cbac1856bd10e7562f86bb759',
+            'upload_date': '20230310',
+            'timestamp': 1678410000,
+            'season_number': 20,
+            'episode_number': 1,
+            'series': 'Top Chef',
+            'episode': 'London Calling',
+            'duration': 3266.03,
+            'season': 'Season 20',
+            'chapters': 'count:7',
+            'thumbnail': r're:^https://.+\.jpg',
+            'age_limit': 14,
+        },
+        'params': {'skip_download': 'm3u8'},
+        'skip': 'This video requires AdobePass MSO credentials',
+    }, {
+        'url': 'https://www.oxygen.com/in-ice-cold-blood/season-1/closing-night',
+        'info_dict': {
+            'id': '3692045',
+            'ext': 'mp4',
+            'title': 'Closing Night',
+            'description': 'md5:3170065c5c2f19548d72a4cbc254af63',
+            'upload_date': '20180401',
+            'timestamp': 1522623600,
+            'season_number': 1,
+            'episode_number': 1,
+            'series': 'In Ice Cold Blood',
+            'episode': 'Closing Night',
+            'duration': 2629.051,
+            'season': 'Season 1',
+            'chapters': 'count:6',
+            'thumbnail': r're:^https://.+\.jpg',
+            'age_limit': 14,
+        },
+        'params': {'skip_download': 'm3u8'},
+        'skip': 'This video requires AdobePass MSO credentials',
     }, {
         'url': 'https://www.oxygen.com/in-ice-cold-blood/season-2/episode-16/videos/handling-the-horwitz-house-after-the-murder-season-2',
+        'info_dict': {
+            'id': '3974019',
+            'ext': 'mp4',
+            'title': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)',
+            'description': 'md5:f9d638dd6946a1c1c0533a9c6100eae5',
+            'upload_date': '20190617',
+            'timestamp': 1560790800,
+            'season_number': 2,
+            'episode_number': 16,
+            'series': 'In Ice Cold Blood',
+            'episode': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)',
+            'duration': 68.235,
+            'season': 'Season 2',
+            'thumbnail': r're:^https://.+\.jpg',
+            'age_limit': 14,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        site, display_id = self._match_valid_url(url).groups()
+        site, display_id = self._match_valid_url(url).group('site', 'id')
         webpage = self._download_webpage(url, display_id)
-        settings = self._parse_json(self._search_regex(
+        settings = self._search_json(
-            r'<script[^>]+data-drupal-selector="drupal-settings-json"[^>]*>({.+?})</script>', webpage, 'drupal settings'),
+            r'<script[^>]+data-drupal-selector="drupal-settings-json"[^>]*>', webpage, 'settings', display_id)
-            display_id)
+        tve = extract_attributes(get_element_html_by_class('tve-video-deck-app', webpage) or '')
-        info = {}
         query = {
-            'mbr': 'true',
+            'manifest': 'm3u',
+            'formats': 'm3u,mpeg4',
         }
-        account_pid, release_pid = [None] * 2
+
-        tve = settings.get('ls_tve')
         if tve:
-            query['manifest'] = 'm3u'
+            account_pid = tve.get('data-mpx-media-account-pid') or 'HNK2IC'
-            mobj = re.search(r'<[^>]+id="pdk-player"[^>]+data-url=["\']?(?:https?:)?//player\.theplatform\.com/p/([^/]+)/(?:[^/]+/)*select/([^?#&"\']+)', webpage)
+            account_id = tve['data-mpx-media-account-id']
-            if mobj:
+            metadata = self._parse_json(
-                account_pid, tp_path = mobj.groups()
+                tve.get('data-normalized-video', ''), display_id, fatal=False, transform_source=unescapeHTML)
-                release_pid = tp_path.strip('/').split('/')[-1]
+            video_id = tve.get('data-guid') or metadata['guid']
-            else:
+            if tve.get('data-entitlement') == 'auth':
-                account_pid = 'HNK2IC'
+                auth = traverse_obj(settings, ('tve_adobe_auth', {dict})) or {}
-                tp_path = release_pid = tve['release_pid']
+                site = remove_end(site, 'tv')
-            if tve.get('entitlement') == 'auth':
+                release_pid = tve['data-release-pid']
-                adobe_pass = settings.get('tve_adobe_auth', {})
-                if site == 'bravotv':
-                    site = 'bravo'
                 resource = self._get_mvpd_resource(
-                    adobe_pass.get('adobePassResourceId') or site,
+                    tve.get('data-adobe-pass-resource-id') or auth.get('adobePassResourceId') or site,
-                    tve['title'], release_pid, tve.get('rating'))
+                    tve['data-title'], release_pid, tve.get('data-rating'))
-                query['auth'] = self._extract_mvpd_auth(
+                query.update({
-                    url, release_pid,
+                    'switch': 'HLSServiceSecure',
-                    adobe_pass.get('adobePassRequestorId') or site, resource)
+                    'auth': self._extract_mvpd_auth(
+                        url, release_pid, auth.get('adobePassRequestorId') or site, resource),
+                })
+
         else:
-            shared_playlist = settings['ls_playlist']
+            ls_playlist = traverse_obj(settings, ('ls_playlist', ..., {dict}), get_all=False) or {}
-            account_pid = shared_playlist['account_pid']
+            account_pid = ls_playlist.get('mpxMediaAccountPid') or 'PHSl-B'
-            metadata = shared_playlist['video_metadata'][shared_playlist['default_clip']]
+            account_id = ls_playlist['mpxMediaAccountId']
-            tp_path = release_pid = metadata.get('release_pid')
+            video_id = ls_playlist['defaultGuid']
-            if not release_pid:
+            metadata = traverse_obj(
-                release_pid = metadata['guid']
+                ls_playlist, ('videos', lambda _, v: v['guid'] == video_id, {dict}), get_all=False)
-                tp_path = 'media/guid/2140479951/' + release_pid
-            info.update({
-                'title': metadata['title'],
-                'description': metadata.get('description'),
-                'season_number': int_or_none(metadata.get('season_num')),
-                'episode_number': int_or_none(metadata.get('episode_num')),
-            })
-            query['switch'] = 'progressive'
-
-        tp_url = 'http://link.theplatform.com/s/%s/%s' % (account_pid, tp_path)
 
+        tp_url = f'https://link.theplatform.com/s/{account_pid}/media/guid/{account_id}/{video_id}'
         tp_metadata = self._download_json(
-            update_url_query(tp_url, {'format': 'preview'}),
+            update_url_query(tp_url, {'format': 'preview'}), video_id, fatal=False)
-            display_id, fatal=False)
-        if tp_metadata:
-            info.update({
-                'title': tp_metadata.get('title'),
-                'description': tp_metadata.get('description'),
-                'duration': float_or_none(tp_metadata.get('duration'), 1000),
-                'season_number': int_or_none(
-                    dict_get(tp_metadata, ('pl1$seasonNumber', 'nbcu$seasonNumber'))),
-                'episode_number': int_or_none(
-                    dict_get(tp_metadata, ('pl1$episodeNumber', 'nbcu$episodeNumber'))),
-                # For some reason the series is sometimes wrapped into a single element array.
-                'series': try_get(
-                    dict_get(tp_metadata, ('pl1$show', 'nbcu$show')),
-                    lambda x: x[0] if isinstance(x, list) else x,
-                    expected_type=str),
-                'episode': dict_get(
-                    tp_metadata, ('pl1$episodeName', 'nbcu$episodeName', 'title')),
-            })
 
-        info.update({
+        seconds_or_none = lambda x: float_or_none(x, 1000)
-            '_type': 'url_transparent',
+        chapters = traverse_obj(tp_metadata, ('chapters', ..., {
-            'id': release_pid,
+            'start_time': ('startTime', {seconds_or_none}),
-            'url': smuggle_url(update_url_query(tp_url, query), {'force_smil_url': True}),
+            'end_time': ('endTime', {seconds_or_none}),
-            'ie_key': 'ThePlatform',
+        }))
-        })
+        # prune pointless single chapters that span the entire duration from short videos
-        return info
+        if len(chapters) == 1 and not traverse_obj(chapters, (0, 'end_time')):
+            chapters = None
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            update_url_query(f'{tp_url}/stream.m3u8', query), video_id, 'mp4', m3u8_id='hls')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'chapters': chapters,
+            **merge_dicts(traverse_obj(tp_metadata, {
+                'title': 'title',
+                'description': 'description',
+                'duration': ('duration', {seconds_or_none}),
+                'timestamp': ('pubDate', {seconds_or_none}),
+                'season_number': (('pl1$seasonNumber', 'nbcu$seasonNumber'), {int_or_none}),
+                'episode_number': (('pl1$episodeNumber', 'nbcu$episodeNumber'), {int_or_none}),
+                'series': (('pl1$show', 'nbcu$show'), (None, ...), {str}),
+                'episode': (('title', 'pl1$episodeNumber', 'nbcu$episodeNumber'), {str_or_none}),
+                'age_limit': ('ratings', ..., 'rating', {parse_age_limit}),
+            }, get_all=False), traverse_obj(metadata, {
+                'title': 'title',
+                'description': 'description',
+                'duration': ('durationInSeconds', {int_or_none}),
+                'timestamp': ('airDate', {unified_timestamp}),
+                'thumbnail': ('thumbnailUrl', {url_or_none}),
+                'season_number': ('seasonNumber', {int_or_none}),
+                'episode_number': ('episodeNumber', {int_or_none}),
+                'episode': 'episodeTitle',
+                'series': 'show',
+            }))
+        }