From ca0be3f1c1e3d452e690e37263b734e788f3438c Mon Sep 17 00:00:00 2001 From: DmitryScaletta Date: Wed, 14 Feb 2024 04:36:08 +0300 Subject: [PATCH] [PromoDJ] Improve extractors --- yt_dlp/extractor/_extractors.py | 2 + yt_dlp/extractor/promodj.py | 110 +++++++++++++++++++++----------- 2 files changed, 76 insertions(+), 36 deletions(-) diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index f35eab137..59873f479 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1537,6 +1537,8 @@ PromoDJUserPageIE, PromoDJBlogPageIE, PromoDJPlaylistIE, + PromoDJMusicPlaylistIE, + PromoDJVideoPlaylistIE, PromoDJIE, PromoDJEmbedIE, PromoDJShortIE, diff --git a/yt_dlp/extractor/promodj.py b/yt_dlp/extractor/promodj.py index e9820b60d..f58be2893 100644 --- a/yt_dlp/extractor/promodj.py +++ b/yt_dlp/extractor/promodj.py @@ -11,7 +11,7 @@ extract_attributes, ExtractorError, get_element_by_class, - get_elements_by_class, + get_elements_html_by_class, int_or_none, parse_duration, traverse_obj, @@ -63,7 +63,7 @@ class PromoDJBaseIE(InfoExtractor): _BASE_URL_RE = r'https?://(?:www\.)?promodj\.com' _MEDIA_TYPES_RE = '|'.join(_MEDIA_TYPES) _NOT_PAGE_RE = '|'.join(['radio', *_PAGES]) - _LOGIN_RE = rf'(?:(?!{_NOT_PAGE_RE}).)[\w-]+' + _LOGIN_RE = rf'(?:(?!{_NOT_PAGE_RE}).)[\w.-]+' def _set_url_page(self, url, page): parsed_url = urllib.parse.urlparse(url) @@ -71,27 +71,25 @@ def _set_url_page(self, url, page): qs['page'] = page return parsed_url._replace(query=urllib.parse.urlencode(qs, doseq=True)).geturl() - def _fetch_page(self, url, parsed_media_types, playlist_id, page): + def _fetch_page(self, url, media_types, playlist_id, page): page_url = self._set_url_page(url, page + 1) html = self._download_webpage(page_url, f'{playlist_id}-page-{page + 1}') current_page = int(clean_html(get_element_by_class('NavigatorCurrentPage', html)) or '1') if current_page != page + 1: return - tracks_dump_html = get_element_by_class('tracks_dump', html) - for item_html in get_elements_by_class('player_standard', tracks_dump_html): - if 'music' in parsed_media_types: - a = get_element_by_class('title', item_html) - if 'video' in parsed_media_types and not a: - a = get_element_by_class('h5videoplayer_promodj_video__title', item_html) - if not a: + for a in get_elements_html_by_class('player_standard_tool__play', html): + url = traverse_obj(extract_attributes(a), ('href', {url_or_none})) + if not url: continue - if url := traverse_obj(extract_attributes(a), ('href', {url_or_none})): + url = url.replace('?play=1', '') + is_video = '/videos/' in url + if is_video and 'video' in media_types or not is_video and 'music' in media_types: yield self.url_result(url, PromoDJIE) def _parse_playlist_links(self, html): PLAYLISTS_RE = r'' - DEFAULT_VIDEO_PLAYLIST_RE = r'
Видео
' + DEFAULT_VIDEO_PLAYLIST_RE = r'
Видео
' playlist_links = [] @@ -210,8 +208,8 @@ def _real_extract(self, url): def entries(): for playlist_url in self._parse_playlist_links(html): - # TODO: parse only music or videos - yield self.url_result(playlist_url, PromoDJPlaylistIE) + ie = PromoDJMusicPlaylistIE if type == 'music' else PromoDJVideoPlaylistIE + yield self.url_result(playlist_url, ie) return self.playlist_result(entries(), playlist_id=page_id) @@ -231,7 +229,6 @@ class PromoDJUserPagesIE(PromoDJBaseIE): def _parse_pages(self, url, playlist_id): html = self._download_webpage(url, playlist_id) content_html = get_element_by_class('dj_universal', get_element_by_class('dj_bblock', html)) - print(re.findall(r'([^<]+)', content_html)) for page_url, page_title in re.findall(r'([^<]+)', content_html): yield self.url_result(page_url, PromoDJUserPageIE, video_title=page_title) @@ -242,9 +239,7 @@ def _fetch_blog_page(self, url, playlist_id, page): if current_page != page + 1: return - for a in get_elements_by_class('post_title', html): - if not a: - continue + for a in get_elements_html_by_class('post_title_moderated', html): if url := traverse_obj(extract_attributes(a), ('href', {url_or_none})): yield self.url_result(url, PromoDJBlogPageIE) @@ -350,6 +345,8 @@ class PromoDJPlaylistIE(PromoDJBaseIE): 'only_matching': True, }] + _MEDIA_TYPES = ['music', 'video'] + def _real_extract(self, url): match = self._match_valid_url(url) login = match.group('login') @@ -358,11 +355,19 @@ def _real_extract(self, url): page_size = self._get_playlist_page_size(url) entries = OnDemandPagedList( - functools.partial(self._fetch_page, url, ['music', 'video'], playlist_id), + functools.partial(self._fetch_page, url, self._MEDIA_TYPES, playlist_id), page_size) return self.playlist_result(entries, playlist_id=playlist_id) +class PromoDJMusicPlaylistIE(PromoDJPlaylistIE): + _MEDIA_TYPES = ['music'] + + +class PromoDJVideoPlaylistIE(PromoDJPlaylistIE): + _MEDIA_TYPES = ['video'] + + class PromoDJIE(PromoDJBaseIE): _VALID_URL = rf'{PromoDJBaseIE._BASE_URL_RE}/{PromoDJBaseIE._LOGIN_RE}/(?P{PromoDJBaseIE._MEDIA_TYPES_RE})/(?P\d+)(?:/\w+)?', _TESTS = [{ @@ -372,9 +377,13 @@ class PromoDJIE(PromoDJBaseIE): 'url': 'https://promodj.com/j-factory/samples/7560171/Amedici_BW1_Intro', 'only_matching': True, }, { - # no download links in html + # music: no download links in html 'url': 'https://promodj.com/gluk/tracks/4713922/DJ_Glyuk_Folk_ing_DJ_Steven_Smile_Remix_2005', 'only_matching': True, + }, { + # video: no download link in html + 'url': 'https://promodj.com/psywanderer/videos/7559147/Chu_de_sa', + 'only_matching': True, }, { # no player 'url': 'https://promodj.com/gluk/tracks/420310/IMpulse_Zakat', @@ -397,20 +406,37 @@ class PromoDJIE(PromoDJBaseIE): }, { 'url': 'https://promodj.com/djperetse/videos/5868236/Fatalist_Project_feat_DJ_Peretse_Den_pobedi_Videoklip', 'only_matching': True, + }, { + # avi + 'url': 'https://promodj.com/djmikis/videos/5311597/Mikis_Live_SDJ_Show', + 'only_matching': True, + }, { + # asf + 'url': 'https://promodj.com/gigsiphonic/videos/7559341/Gigsiphonic_PODCAST_309_Extended_video_version', + 'only_matching': True, + }, { + # not valid html + 'url': 'https://promodj.com/martin.sehnal/videos/7555841/Martin_Sehnal_CII_33_Plus_CII_32_Clothes_on_the_peg_2_020_2_024_02_01th', + 'only_matching': True, }] _IS_PAID_RE = r'Цена:' - # examples: MP3, 320 Кбит | MP4, 20157 Кбит | WAV, 1412 Кбит - _FORMATS_RE = r'[^\"]+\.(?:mp3|mp4|wav))\">\s*(?PMP3|MP4|WAV), (?P\d+) Кбит\s*' + # examples: MP3, 320 Кбит | MP4, 20157 Кбит | WAV, 1412 Кбит | AVI, 1731 Кбит | ASF, 6905 Кбит + _FORMATS_RE = r'[^\"]+)\">\s*(?P\w+), (?P\d+) Кбит\s*' _VIEW_COUNT_RE = r'(?:Прослушиваний|Просмотров):\s*(\d+)' - # examples: 0:21, 1:07, 74:38 - _DURATION_RE = r'Продолжительность:\s*(\d{1,}:\d{2})' - # examples: 818.4 Кб, 12.9 Мб, 4 Гб, 1.76 Гб - _SIZE_RE = r'Размер:\s*(?P\d{1,3}(?:\.\d{1,2})?)\s*(?PКб|Мб|Гб)' - # examples: сегодня 2:55, вчера 23:17, 1 июня 2016 3:46 + # examples: 0:21 | 1:07 | 74:38 + _DURATION_RE = r'Продолжительность:\s*(\d+:\d{2})' + # examples: 818.4 Кб | 12.9 Мб | 4 Гб | 1.76 Гб | 1001.5 Мб + _SIZE_RE = r'Размер:\s*(?P\d+(?:\.\d+)?)\s*(?PКб|Мб|Гб)' + # examples: сегодня 2:55 | вчера 23:17 | 1 июня 2016 3:46 _TIMESTAMP_RE = r'Публикация:\s*(?Pвчера|сегодня|\d{1,2})(?: (?P[а-я]+) (?P\d{4}))?\s*(?P\d{1,2}):(?P\d{2})' _TAGS_RE = r'([^\n]+)' + # https://regex101.com/r/2ZkUmW/1 + _MUSIC_DATA_REGEX = r'({\"no_preroll\":false,\"seekAny\":true,\"sources\":[^\n]+)\);' + # https://regex101.com/r/b9utBf/1 + _VIDEO_DATA_REGEX = r'({\"video\":true,\"config\":[^\n]+)\);' + def _parse_ru_date(self, raw_date): RU_MONTHS = ['января', 'февраля', 'марта', 'апреля', 'мая', 'июня', 'июля', 'августа', 'сентября', 'октября', 'ноября', 'декабря'] day, month, year, hours, minutes = raw_date @@ -435,20 +461,32 @@ def _parse_ru_size(self, raw_size): size, size_unit = raw_size return int(float(size) * pow(1024, RU_SIZE_UNITS.index(size_unit))) - def _parse_media(self, html, id): - meta_html = get_element_by_class('clearfix', get_element_by_class('dj_bblock', html)) + def _parse_media(self, html, id, type): + # html can be invalid + try: + meta_html = get_elements_html_by_class('dj_universal', html)[1] + except Exception: + meta_html = html - is_paid = re.search(self._IS_PAID_RE, meta_html) formats_from_html = re.findall(self._FORMATS_RE, meta_html) + has_formats = len(formats_from_html) != 0 + is_paid = re.search(self._IS_PAID_RE, meta_html) - if is_paid or len(formats_from_html) == 0: - media_data_raw = self._search_regex( - r'({\"no_preroll\":false,\"seekAny\":true,\"sources\":[^\n]+)\);', html, 'media data') + if not has_formats and is_paid: + media_data_raw = self._search_regex(self._MUSIC_DATA_REGEX, html, 'media data') media_data = self._parse_json(media_data_raw, id) formats = [{ 'url': source.get('URL'), 'size': int_or_none(source.get('size')), } for source in traverse_obj(media_data, ('sources')) if url_or_none(source.get('URL'))] + elif not has_formats and type == 'videos': + media_data_raw = self._search_regex(self._VIDEO_DATA_REGEX, html, 'media data') + media_data = self._parse_json(media_data_raw, id) + video_config = self._parse_json(media_data['config'], id) + video = traverse_obj(video_config, ('playlist', 'item', 0)) + formats = [{ + 'url': traverse_obj(video, ('play', '@url', {url_or_none})), + }] else: formats = [{ 'url': url, @@ -462,16 +500,16 @@ def _parse_media(self, html, id): 'id': id, 'title': clean_html(get_element_by_class('file_title', html)), 'formats': formats, - 'view_count': int_or_none(self._search_regex(self._VIEW_COUNT_RE, meta_html, 'view_count')), + 'view_count': int_or_none(self._search_regex(self._VIEW_COUNT_RE, meta_html, 'view_count', default=None)), 'duration': parse_duration(self._search_regex(self._DURATION_RE, meta_html, 'duration')), 'timestamp': self._parse_ru_date(re.findall(self._TIMESTAMP_RE, meta_html)[0]), 'tags': self._html_search_regex(self._TAGS_RE, meta_html, 'tags').split(', '), } def _real_extract(self, url): - id = self._match_id(url) + type, id = self._match_valid_url(url).groups() html = self._download_webpage(url, id) - return self._parse_media(html, id) + return self._parse_media(html, id, type) class PromoDJEmbedIE(PromoDJBaseIE):