From 7d0f6f0c4527aa1c1f99984c5b34d21ebc87228d Mon Sep 17 00:00:00 2001 From: Burve Date: Tue, 26 Jul 2022 06:41:52 +0300 Subject: [PATCH] [extractor/Crunchyroll] Handle missing metadata correctly (#4405) Closes #4399 Authored by pukkandan, Burve --- yt_dlp/extractor/crunchyroll.py | 58 ++++++++++++++------------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py index d5aa45ff8..9dda53c68 100644 --- a/yt_dlp/extractor/crunchyroll.py +++ b/yt_dlp/extractor/crunchyroll.py @@ -813,56 +813,36 @@ def _real_extract(self, url): episode_response = self._download_json( f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id, - note='Retrieving episode metadata', - query=params) + note='Retrieving episode metadata', query=params) if episode_response.get('is_premium_only') and not episode_response.get('playback'): raise ExtractorError('This video is for premium members only.', expected=True) - stream_response = self._download_json( - episode_response['playback'], display_id, - note='Retrieving stream info') - thumbnails = [] - for thumbnails_data in traverse_obj(episode_response, ('images', 'thumbnail')): - for thumbnail_data in thumbnails_data: - thumbnails.append({ - 'url': thumbnail_data.get('source'), - 'width': thumbnail_data.get('width'), - 'height': thumbnail_data.get('height'), - }) - subtitles = {} - for lang, subtitle_data in stream_response.get('subtitles').items(): - subtitles[lang] = [{ - 'url': subtitle_data.get('url'), - 'ext': subtitle_data.get('format') - }] + stream_response = self._download_json(episode_response['playback'], display_id, note='Retrieving stream info') + get_streams = lambda name: (traverse_obj(stream_response, name) or {}).items() requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])] hardsub_preference = qualities(requested_hardsubs[::-1]) requested_formats = self._configuration_arg('format') or ['adaptive_hls'] formats = [] - for stream_type, streams in stream_response.get('streams', {}).items(): + for stream_type, streams in get_streams('streams'): if stream_type not in requested_formats: continue for stream in streams.values(): hardsub_lang = stream.get('hardsub_locale') or '' if hardsub_lang.lower() not in requested_hardsubs: continue - format_id = join_nonempty( - stream_type, - format_field(stream, 'hardsub_locale', 'hardsub-%s')) + format_id = join_nonempty(stream_type, format_field(stream, 'hardsub_locale', 'hardsub-%s')) if not stream.get('url'): continue - if stream_type.split('_')[-1] == 'hls': + if stream_type.endswith('hls'): adaptive_formats = self._extract_m3u8_formats( stream['url'], display_id, 'mp4', m3u8_id=format_id, - note='Downloading %s information' % format_id, - fatal=False) - elif stream_type.split('_')[-1] == 'dash': + fatal=False, note=f'Downloading {format_id} HLS manifest') + elif stream_type.endswith('dash'): adaptive_formats = self._extract_mpd_formats( stream['url'], display_id, mpd_id=format_id, - note='Downloading %s information' % format_id, - fatal=False) + fatal=False, note=f'Downloading {format_id} MPD manifest') for f in adaptive_formats: if f.get('acodec') != 'none': f['language'] = stream_response.get('audio_locale') @@ -872,10 +852,10 @@ def _real_extract(self, url): return { 'id': internal_id, - 'title': '%s Episode %s – %s' % (episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')), - 'description': episode_response.get('description').replace(r'\r\n', '\n'), + 'title': '%s Episode %s – %s' % ( + episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')), + 'description': try_get(episode_response, lambda x: x['description'].replace(r'\r\n', '\n')), 'duration': float_or_none(episode_response.get('duration_ms'), 1000), - 'thumbnails': thumbnails, 'series': episode_response.get('series_title'), 'series_id': episode_response.get('series_id'), 'season': episode_response.get('season_title'), @@ -883,8 +863,18 @@ def _real_extract(self, url): 'season_number': episode_response.get('season_number'), 'episode': episode_response.get('title'), 'episode_number': episode_response.get('sequence_number'), - 'subtitles': subtitles, - 'formats': formats + 'formats': formats, + 'thumbnails': [{ + 'url': thumb.get('source'), + 'width': thumb.get('width'), + 'height': thumb.get('height'), + } for thumb in traverse_obj(episode_response, ('images', 'thumbnail', ..., ...)) or []], + 'subtitles': { + lang: [{ + 'url': subtitle_data.get('url'), + 'ext': subtitle_data.get('format') + }] for lang, subtitle_data in get_streams('subtitles') + }, }