From df2c44a034ad40e01d55d26e322edecd1552d215 Mon Sep 17 00:00:00 2001 From: bergoid Date: Tue, 28 Nov 2023 19:27:01 +0100 Subject: [PATCH] Line length --- yt_dlp/extractor/vrt.py | 348 +++++++++++++++++++++++++--------------- 1 file changed, 223 insertions(+), 125 deletions(-) diff --git a/yt_dlp/extractor/vrt.py b/yt_dlp/extractor/vrt.py index 8093f78a1..7bf39a7c4 100644 --- a/yt_dlp/extractor/vrt.py +++ b/yt_dlp/extractor/vrt.py @@ -30,34 +30,46 @@ class VRTBaseIE(InfoExtractor): self.report_drm(video_id) formats, subtitles = [], {} - for target in traverse_obj(data, ('targetUrls', lambda _, v: url_or_none(v['url']) and v['type'])): + for target in traverse_obj( + data, ('targetUrls', lambda _, v: url_or_none(v['url']) and v['type']) + ): format_type = target['type'].upper() format_url = target['url'] if format_type in ('HLS', 'HLS_AES'): fmts, subs = self._extract_m3u8_formats_and_subtitles( - format_url, video_id, 'mp4', m3u8_id=format_type, fatal=False) + format_url, video_id, 'mp4', m3u8_id=format_type, fatal=False + ) formats.extend(fmts) self._merge_subtitles(subs, target=subtitles) elif format_type == 'HDS': - formats.extend(self._extract_f4m_formats( - format_url, video_id, f4m_id=format_type, fatal=False)) + formats.extend( + self._extract_f4m_formats( + format_url, video_id, f4m_id=format_type, fatal=False + ) + ) elif format_type == 'MPEG_DASH': fmts, subs = self._extract_mpd_formats_and_subtitles( - format_url, video_id, mpd_id=format_type, fatal=False) + format_url, video_id, mpd_id=format_type, fatal=False + ) formats.extend(fmts) self._merge_subtitles(subs, target=subtitles) elif format_type == 'HSS': fmts, subs = self._extract_ism_formats_and_subtitles( - format_url, video_id, ism_id='mss', fatal=False) + format_url, video_id, ism_id='mss', fatal=False + ) formats.extend(fmts) self._merge_subtitles(subs, target=subtitles) else: - formats.append({ - 'format_id': format_type, - 'url': format_url, - }) + formats.append( + { + 'format_id': format_type, + 'url': format_url, + } + ) - for sub in traverse_obj(data, ('subtitleUrls', lambda _, v: v['url'] and v['type'] == 'CLOSED')): + for sub in traverse_obj( + data, ('subtitleUrls', lambda _, v: v['url'] and v['type'] == 'CLOSED') + ): subtitles.setdefault('nl', []).append({'url': sub['url']}) return formats, subtitles @@ -65,37 +77,60 @@ class VRTBaseIE(InfoExtractor): def _call_api(self, video_id, client='null', id_token=None, version='v2'): json_response = self._download_json( f'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/{version}/tokens', - None, 'Downloading player token', 'Failed to download player token', + None, + 'Downloading player token', + 'Failed to download player token', headers={'Content-Type': 'application/json'}, - data=json.dumps({'identityToken': id_token or self._get_cookies('https://www.vrt.be').get('vrtnu-site_profile_vt').value}).encode()) + data=json.dumps( + { + 'identityToken': id_token + or self._get_cookies('https://www.vrt.be') + .get('vrtnu-site_profile_vt') + .value + } + ).encode(), + ) player_token = json_response['vrtPlayerToken'] return self._download_json( f'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/{version}/videos/{video_id}', - video_id, 'Downloading API JSON', 'Failed to download API JSON', query={ + video_id, + 'Downloading API JSON', + 'Failed to download API JSON', + query={ 'vrtPlayerToken': player_token, 'client': client, - }) + }, + ) class VRTLoginIE(VRTBaseIE): - _NETRC_MACHINE = 'vrtnu' _authenticated = False def _perform_login(self, username, password): - - self._request_webpage('https://www.vrt.be/vrtnu/sso/login', None, note='Getting session cookies', errnote='Failed to get session cookies') + self._request_webpage( + 'https://www.vrt.be/vrtnu/sso/login', + None, + note='Getting session cookies', + errnote='Failed to get session cookies', + ) self._download_json( - 'https://login.vrt.be/perform_login', None, data=json.dumps({ - 'loginID': username, - 'password': password, - 'clientId': 'vrtnu-site' - }).encode(), headers={ + 'https://login.vrt.be/perform_login', + None, + data=json.dumps( + {'loginID': username, 'password': password, 'clientId': 'vrtnu-site'} + ).encode(), + headers={ 'Content-Type': 'application/json', - 'Oidcxsrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value, - }, note='Logging in', errnote='Login failed') + 'Oidcxsrf': self._get_cookies('https://login.vrt.be') + .get('OIDCXSRF') + .value, + }, + note='Logging in', + errnote='Login failed', + ) self._authenticated = True return @@ -103,29 +138,32 @@ class VRTLoginIE(VRTBaseIE): class VRTIE(VRTLoginIE): IE_DESC = 'VRT NWS, Flanders News, Flandern Info and Sporza' _VALID_URL = r'https?://(?:www\.)?(?Pvrt\.be/vrtnws|sporza\.be)/[a-z]{2}/\d{4}/\d{2}/\d{2}/(?P[^/?&#]+)' - _TESTS = [{ - 'url': 'https://www.vrt.be/vrtnws/nl/2019/05/15/beelden-van-binnenkant-notre-dame-een-maand-na-de-brand/', - 'info_dict': { - 'id': 'pbs-pub-7855fc7b-1448-49bc-b073-316cb60caa71$vid-2ca50305-c38a-4762-9890-65cbd098b7bd', - 'ext': 'mp4', - 'title': 'Beelden van binnenkant Notre-Dame, één maand na de brand', - 'description': 'md5:6fd85f999b2d1841aa5568f4bf02c3ff', - 'duration': 31.2, - 'thumbnail': 'https://images.vrt.be/orig/2019/05/15/2d914d61-7710-11e9-abcc-02b7b76bf47f.jpg', + _TESTS = [ + { + 'url': 'https://www.vrt.be/vrtnws/nl/2019/05/15/beelden-van-binnenkant-notre-dame-een-maand-na-de-brand/', + 'info_dict': { + 'id': 'pbs-pub-7855fc7b-1448-49bc-b073-316cb60caa71$vid-2ca50305-c38a-4762-9890-65cbd098b7bd', + 'ext': 'mp4', + 'title': 'Beelden van binnenkant Notre-Dame, één maand na de brand', + 'description': 'md5:6fd85f999b2d1841aa5568f4bf02c3ff', + 'duration': 31.2, + 'thumbnail': 'https://images.vrt.be/orig/2019/05/15/2d914d61-7710-11e9-abcc-02b7b76bf47f.jpg', + }, + 'params': {'skip_download': 'm3u8'}, }, - 'params': {'skip_download': 'm3u8'}, - }, { - 'url': 'https://sporza.be/nl/2019/05/15/de-belgian-cats-zijn-klaar-voor-het-ek/', - 'info_dict': { - 'id': 'pbs-pub-e1d6e4ec-cbf4-451e-9e87-d835bb65cd28$vid-2ad45eb6-9bc8-40d4-ad72-5f25c0f59d75', - 'ext': 'mp4', - 'title': 'De Belgian Cats zijn klaar voor het EK', - 'description': 'Video: De Belgian Cats zijn klaar voor het EK mét Ann Wauters | basketbal, sport in het journaal', - 'duration': 115.17, - 'thumbnail': 'https://images.vrt.be/orig/2019/05/15/11c0dba3-770e-11e9-abcc-02b7b76bf47f.jpg', + { + 'url': 'https://sporza.be/nl/2019/05/15/de-belgian-cats-zijn-klaar-voor-het-ek/', + 'info_dict': { + 'id': 'pbs-pub-e1d6e4ec-cbf4-451e-9e87-d835bb65cd28$vid-2ad45eb6-9bc8-40d4-ad72-5f25c0f59d75', + 'ext': 'mp4', + 'title': 'De Belgian Cats zijn klaar voor het EK', + 'description': 'Video: De Belgian Cats zijn klaar voor het EK mét Ann Wauters | basketbal, sport in het journaal', + 'duration': 115.17, + 'thumbnail': 'https://images.vrt.be/orig/2019/05/15/11c0dba3-770e-11e9-abcc-02b7b76bf47f.jpg', + }, + 'params': {'skip_download': 'm3u8'}, }, - 'params': {'skip_download': 'm3u8'}, - }] + ] _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy' _CONTEXT_ID = 'R3595707040' _REST_API_BASE_TOKEN = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2' @@ -141,16 +179,22 @@ class VRTIE(VRTLoginIE): attrs = extract_attributes(get_element_html_by_class('vrtvideo', webpage) or '') asset_id = attrs.get('data-video-id') or attrs['data-videoid'] - publication_id = traverse_obj(attrs, 'data-publication-id', 'data-publicationid') + publication_id = traverse_obj( + attrs, 'data-publication-id', 'data-publicationid' + ) if publication_id: asset_id = f'{publication_id}${asset_id}' - client = traverse_obj(attrs, 'data-client-code', 'data-client') or self._CLIENT_MAP[site] + client = ( + traverse_obj(attrs, 'data-client-code', 'data-client') + or self._CLIENT_MAP[site] + ) data = self._call_api(asset_id, client) formats, subtitles = self._extract_formats_and_subtitles(data, asset_id) description = self._html_search_meta( - ['og:description', 'twitter:description', 'description'], webpage) + ['og:description', 'twitter:description', 'description'], webpage + ) if description == '…': description = None @@ -162,18 +206,26 @@ class VRTIE(VRTLoginIE): 'thumbnail': url_or_none(attrs.get('data-posterimage')), 'duration': float_or_none(attrs.get('data-duration'), 1000), '_old_archive_ids': [make_archive_id('Canvas', asset_id)], - **traverse_obj(data, { - 'title': ('title', {str}), - 'description': ('shortDescription', {str}), - 'duration': ('duration', {functools.partial(float_or_none, scale=1000)}), - 'thumbnail': ('posterImageUrl', {url_or_none}), - }), + **traverse_obj( + data, + { + 'title': ('title', {str}), + 'description': ('shortDescription', {str}), + 'duration': ( + 'duration', + {functools.partial(float_or_none, scale=1000)}, + ), + 'thumbnail': ('posterImageUrl', {url_or_none}), + }, + ), } class VrtNUIE(VRTLoginIE): IE_DESC = 'VRT MAX' - _VALID_URL = r'https?://(?:www\.)?vrt\.be/(vrtmax|vrtnu)/a-z/(?:[^/]+/){2}(?P[^/?#&]+)' + _VALID_URL = ( + r'https?://(?:www\.)?vrt\.be/(vrtmax|vrtnu)/a-z/(?:[^/]+/){2}(?P[^/?#&]+)' + ) _TESTS = [ { 'url': 'https://www.vrt.be/vrtmax/a-z/pano/trailer/pano-trailer-najaar-2023/', @@ -218,7 +270,7 @@ class VrtNUIE(VRTLoginIE): 'thumbnail': 'https://images.vrt.be/orig/2023/11/07/37d244f0-7d8a-11ee-91d7-02b7b76bf47f.jpg', 'ext': 'mp4', }, - } + }, ] _NETRC_MACHINE = 'vrtnu' @@ -229,13 +281,31 @@ class VrtNUIE(VRTLoginIE): display_id = self._match_id(url) parsed_url = urllib.parse.urlparse(url) - self._request_webpage('https://www.vrt.be/vrtnu/sso/login', None, note='Getting tokens', errnote='Failed to get tokens') + self._request_webpage( + 'https://www.vrt.be/vrtnu/sso/login', + None, + note='Getting tokens', + errnote='Failed to get tokens', + ) metadata = self._download_json( 'https://www.vrt.be/vrtnu-api/graphql/v1', - display_id, 'Downloading asset JSON', 'Unable to download asset JSON', - headers={'Content-Type': 'application/json', 'Authorization': f'Bearer {self._get_cookies("https://www.vrt.be").get("vrtnu-site_profile_at").value}'}, - data=json.dumps({'operationName': 'VideoPage', 'query': self._VIDEOPAGE_QUERY, 'variables': {'pageId': f'{parsed_url.path.rstrip("/")}.model.json'}}).encode() + display_id, + 'Downloading asset JSON', + 'Unable to download asset JSON', + headers={ + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self._get_cookies("https://www.vrt.be").get("vrtnu-site_profile_at").value}', + }, + data=json.dumps( + { + 'operationName': 'VideoPage', + 'query': self._VIDEOPAGE_QUERY, + 'variables': { + 'pageId': f'{parsed_url.path.rstrip("/")}.model.json' + }, + } + ).encode(), )['data']['page'] video_id = metadata['episode']['watchAction']['streamId'] @@ -245,28 +315,41 @@ class VrtNUIE(VRTLoginIE): ld_json = {} streaming_info = self._call_api(video_id, client='vrtnu-web@PROD') - formats, subtitles = self._extract_formats_and_subtitles(streaming_info, video_id) + formats, subtitles = self._extract_formats_and_subtitles( + streaming_info, video_id + ) return { - **traverse_obj(metadata, { - 'title': ('seo', 'title', {str_or_none}), - 'season_number': ('episode', 'onTimeRaw', {lambda x: x[:4]}, {int_or_none}), - 'description': ('seo', 'description', {str_or_none}), - 'timestamp': ('episode', 'onTimeRaw', {parse_iso8601}), - 'release_timestamp': ('episode', 'onTimeRaw', {parse_iso8601}), - 'release_date': ('episode', 'onTimeRaw', {unified_strdate}), - 'upload_date': ('episode', 'onTimeRaw', {unified_strdate}), - 'series': ('episode', 'program', 'title'), - 'episode': ('episode', 'episodeNumberRaw', {str_or_none}), - 'episode_number': ('episode', 'episodeNumberRaw', {int_or_none}), - 'age_limit': ('episode', 'ageRaw', {parse_age_limit}), - 'display_id': ('episode', 'name', {parse_age_limit}), - }), - **traverse_obj(ld_json, { - 'season': ('partOfSeason', 'name'), - 'season_id': ('partOfSeason', '@id'), - 'episode_id': ('@id', {str_or_none}), - }), + **traverse_obj( + metadata, + { + 'title': ('seo', 'title', {str_or_none}), + 'season_number': ( + 'episode', + 'onTimeRaw', + {lambda x: x[:4]}, + {int_or_none}, + ), + 'description': ('seo', 'description', {str_or_none}), + 'timestamp': ('episode', 'onTimeRaw', {parse_iso8601}), + 'release_timestamp': ('episode', 'onTimeRaw', {parse_iso8601}), + 'release_date': ('episode', 'onTimeRaw', {unified_strdate}), + 'upload_date': ('episode', 'onTimeRaw', {unified_strdate}), + 'series': ('episode', 'program', 'title'), + 'episode': ('episode', 'episodeNumberRaw', {str_or_none}), + 'episode_number': ('episode', 'episodeNumberRaw', {int_or_none}), + 'age_limit': ('episode', 'ageRaw', {parse_age_limit}), + 'display_id': ('episode', 'name', {parse_age_limit}), + }, + ), + **traverse_obj( + ld_json, + { + 'season': ('partOfSeason', 'name'), + 'season_id': ('partOfSeason', '@id'), + 'episode_id': ('@id', {str_or_none}), + }, + ), 'id': video_id, 'channel': 'VRT', 'formats': formats, @@ -279,26 +362,30 @@ class VrtNUIE(VRTLoginIE): class KetnetIE(VRTBaseIE): _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?P(?:[^/]+/)*[^/?#&]+)' - _TESTS = [{ - 'url': 'https://www.ketnet.be/kijken/m/meisjes/6/meisjes-s6a5', - 'info_dict': { - 'id': 'pbs-pub-39f8351c-a0a0-43e6-8394-205d597d6162$vid-5e306921-a9aa-4fa9-9f39-5b82c8f1028e', - 'ext': 'mp4', - 'title': 'Meisjes', - 'episode': 'Reeks 6: Week 5', - 'season': 'Reeks 6', - 'series': 'Meisjes', - 'timestamp': 1685251800, - 'upload_date': '20230528', - }, - 'params': {'skip_download': 'm3u8'}, - }] + _TESTS = [ + { + 'url': 'https://www.ketnet.be/kijken/m/meisjes/6/meisjes-s6a5', + 'info_dict': { + 'id': 'pbs-pub-39f8351c-a0a0-43e6-8394-205d597d6162$vid-5e306921-a9aa-4fa9-9f39-5b82c8f1028e', + 'ext': 'mp4', + 'title': 'Meisjes', + 'episode': 'Reeks 6: Week 5', + 'season': 'Reeks 6', + 'series': 'Meisjes', + 'timestamp': 1685251800, + 'upload_date': '20230528', + }, + 'params': {'skip_download': 'm3u8'}, + } + ] def _real_extract(self, url): display_id = self._match_id(url) video = self._download_json( - 'https://senior-bff.ketnet.be/graphql', display_id, query={ + 'https://senior-bff.ketnet.be/graphql', + display_id, + query={ 'query': '''{ video(id: "content/ketnet/nl/%s.model.json") { description @@ -311,8 +398,10 @@ class KetnetIE(VRTBaseIE): subtitleVideodetail titleVideodetail } -}''' % display_id, - })['data']['video'] +}''' + % display_id, + }, + )['data']['video'] video_id = urllib.parse.unquote(video['mediaReference']) data = self._call_api(video_id, 'ketnet@PROD', version='v1') @@ -323,39 +412,45 @@ class KetnetIE(VRTBaseIE): 'formats': formats, 'subtitles': subtitles, '_old_archive_ids': [make_archive_id('Canvas', video_id)], - **traverse_obj(video, { - 'title': ('titleVideodetail', {str}), - 'description': ('description', {str}), - 'thumbnail': ('thumbnail', {url_or_none}), - 'timestamp': ('publicationDate', {parse_iso8601}), - 'series': ('programTitle', {str}), - 'season': ('seasonTitle', {str}), - 'episode': ('subtitleVideodetail', {str}), - 'episode_number': ('episodeNr', {int_or_none}), - }), + **traverse_obj( + video, + { + 'title': ('titleVideodetail', {str}), + 'description': ('description', {str}), + 'thumbnail': ('thumbnail', {url_or_none}), + 'timestamp': ('publicationDate', {parse_iso8601}), + 'series': ('programTitle', {str}), + 'season': ('seasonTitle', {str}), + 'episode': ('subtitleVideodetail', {str}), + 'episode_number': ('episodeNr', {int_or_none}), + }, + ), } class DagelijkseKostIE(VRTBaseIE): IE_DESC = 'dagelijksekost.een.be' _VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P[^/?#&]+)' - _TESTS = [{ - 'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof', - 'info_dict': { - 'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa', - 'ext': 'mp4', - 'title': 'Hachis parmentier met witloof', - 'description': 'md5:9960478392d87f63567b5b117688cdc5', - 'display_id': 'hachis-parmentier-met-witloof', - }, - 'params': {'skip_download': 'm3u8'}, - }] + _TESTS = [ + { + 'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof', + 'info_dict': { + 'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa', + 'ext': 'mp4', + 'title': 'Hachis parmentier met witloof', + 'description': 'md5:9960478392d87f63567b5b117688cdc5', + 'display_id': 'hachis-parmentier-met-witloof', + }, + 'params': {'skip_download': 'm3u8'}, + } + ] def _real_extract(self, url): display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) video_id = self._html_search_regex( - r'data-url=(["\'])(?P(?:(?!\1).)+)\1', webpage, 'video id', group='id') + r'data-url=(["\'])(?P(?:(?!\1).)+)\1', webpage, 'video id', group='id' + ) data = self._call_api(video_id, 'dako@prod', version='v1') formats, subtitles = self._extract_formats_and_subtitles(data, video_id) @@ -365,10 +460,13 @@ class DagelijkseKostIE(VRTBaseIE): 'formats': formats, 'subtitles': subtitles, 'display_id': display_id, - 'title': strip_or_none(get_element_by_class( - 'dish-metadata__title', webpage) or self._html_search_meta('twitter:title', webpage)), - 'description': clean_html(get_element_by_class( - 'dish-description', webpage)) or self._html_search_meta( - ['description', 'twitter:description', 'og:description'], webpage), + 'title': strip_or_none( + get_element_by_class('dish-metadata__title', webpage) + or self._html_search_meta('twitter:title', webpage) + ), + 'description': clean_html(get_element_by_class('dish-description', webpage)) + or self._html_search_meta( + ['description', 'twitter:description', 'og:description'], webpage + ), '_old_archive_ids': [make_archive_id('Canvas', video_id)], }