[extractor/zenyandex] Fix extractors (#3750, #5268)

Closes #3736
Authored by:  lksj, puc9, pukkandan

Co-authored-by: puc9 <51006296+puc9@users.noreply.github.com>
This commit is contained in:
Alex Karabanov 2022-10-25 14:20:48 +04:00 committed by GitHub
parent c66ed4e2e5
commit c9bd65185c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 129 additions and 45 deletions

View File

@ -122,7 +122,8 @@ def print_skipping(reason):
params['outtmpl'] = tname + '_' + params['outtmpl'] params['outtmpl'] = tname + '_' + params['outtmpl']
if is_playlist and 'playlist' not in test_case: if is_playlist and 'playlist' not in test_case:
params.setdefault('extract_flat', 'in_playlist') params.setdefault('extract_flat', 'in_playlist')
params.setdefault('playlistend', test_case.get('playlist_mincount')) params.setdefault('playlistend', test_case.get(
'playlist_mincount', test_case.get('playlist_count', -2) + 1))
params.setdefault('skip_download', True) params.setdefault('skip_download', True)
ydl = YoutubeDL(params, auto_init=False) ydl = YoutubeDL(params, auto_init=False)

View File

@ -1,5 +1,4 @@
import itertools import itertools
import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
@ -7,6 +6,8 @@
extract_attributes, extract_attributes,
int_or_none, int_or_none,
lowercase_escape, lowercase_escape,
parse_qs,
traverse_obj,
try_get, try_get,
url_or_none, url_or_none,
) )
@ -23,7 +24,6 @@ class YandexVideoIE(InfoExtractor):
''' '''
_TESTS = [{ _TESTS = [{
'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374', 'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374',
'md5': 'e02a05bfaf0d9615ef07ae3a10f4faf4',
'info_dict': { 'info_dict': {
'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374', 'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374',
'ext': 'mp4', 'ext': 'mp4',
@ -38,6 +38,7 @@ class YandexVideoIE(InfoExtractor):
'like_count': int, 'like_count': int,
'dislike_count': int, 'dislike_count': int,
}, },
'params': {'skip_download': 'm3u8'},
}, { }, {
'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda', 'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda',
'only_matching': True, 'only_matching': True,
@ -188,34 +189,35 @@ def _real_extract(self, url):
class ZenYandexIE(InfoExtractor): class ZenYandexIE(InfoExtractor):
_VALID_URL = r'https?://zen\.yandex\.ru(?:/video)?/(media|watch)/(?:(?:id/[^/]+/|[^/]+/)(?:[a-z0-9-]+)-)?(?P<id>[a-z0-9-]+)' _VALID_URL = r'https?://(zen\.yandex|dzen)\.ru(?:/video)?/(media|watch)/(?:(?:id/[^/]+/|[^/]+/)(?:[a-z0-9-]+)-)?(?P<id>[a-z0-9-]+)'
_TESTS = [{ _TESTS = [{
'url': 'https://zen.yandex.ru/media/popmech/izverjenie-vulkana-iz-spichek-zreliscnyi-opyt-6002240ff8b1af50bb2da5e3',
'info_dict': {
'id': '6002240ff8b1af50bb2da5e3',
'ext': 'mp4',
'title': 'Извержение вулкана из спичек: зрелищный опыт',
'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
'thumbnail': 're:^https://avatars.mds.yandex.net/',
'uploader': 'Популярная механика',
},
'params': {
'skip_download': 'm3u8',
},
}, {
'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7', 'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
'info_dict': { 'info_dict': {
'id': '60c7c443da18892ebfe85ed7', 'id': '60c7c443da18892ebfe85ed7',
'ext': 'mp4', 'ext': 'mp4',
'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах', 'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
'description': 'md5:f3db3d995763b9bbb7b56d4ccdedea89', 'description': 'md5:f3db3d995763b9bbb7b56d4ccdedea89',
'thumbnail': 're:^https://avatars.mds.yandex.net/', 'thumbnail': 're:^https://avatars.dzeninfra.ru/',
'uploader': 'AcademeG DailyStream' 'uploader': 'AcademeG DailyStream'
}, },
'params': { 'params': {
'skip_download': 'm3u8', 'skip_download': 'm3u8',
'format': 'bestvideo', 'format': 'bestvideo',
}, },
'skip': 'The page does not exist',
}, {
'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
'info_dict': {
'id': '60c7c443da18892ebfe85ed7',
'ext': 'mp4',
'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
'description': 'md5:f3db3d995763b9bbb7b56d4ccdedea89',
'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
'uploader': 'AcademeG DailyStream',
'upload_date': '20191111',
'timestamp': 1573465585,
},
'params': {'skip_download': 'm3u8'},
}, { }, {
'url': 'https://zen.yandex.ru/video/watch/6002240ff8b1af50bb2da5e3', 'url': 'https://zen.yandex.ru/video/watch/6002240ff8b1af50bb2da5e3',
'info_dict': { 'info_dict': {
@ -223,21 +225,42 @@ class ZenYandexIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'Извержение вулкана из спичек: зрелищный опыт', 'title': 'Извержение вулкана из спичек: зрелищный опыт',
'description': 'md5:053ad3c61b5596d510c9a199dc8ee633', 'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
'uploader': 'Популярная механика', 'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
'uploader': 'TechInsider',
'timestamp': 1611378221,
'upload_date': '20210123',
}, },
'params': { 'params': {'skip_download': 'm3u8'},
'skip_download': 'm3u8', }, {
'url': 'https://dzen.ru/video/watch/6002240ff8b1af50bb2da5e3',
'info_dict': {
'id': '6002240ff8b1af50bb2da5e3',
'ext': 'mp4',
'title': 'Извержение вулкана из спичек: зрелищный опыт',
'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
'thumbnail': 're:^https://avatars.dzeninfra.ru/',
'uploader': 'TechInsider',
'upload_date': '20210123',
'timestamp': 1611378221,
}, },
'params': {'skip_download': 'm3u8'},
}, { }, {
'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360', 'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
'only_matching': True, 'only_matching': True,
}, {
'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
'only_matching': True,
}] }]
def _real_extract(self, url): def _real_extract(self, url):
id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage(url, id) webpage = self._download_webpage(url, video_id)
data_json = self._parse_json( redirect = self._search_json(r'var it\s*=\s*', webpage, 'redirect', id, default={}).get('retpath')
self._search_regex(r'data\s*=\s*({["\']_*serverState_*video.+?});', webpage, 'metadata'), id) if redirect:
video_id = self._match_id(redirect)
webpage = self._download_webpage(redirect, video_id, note='Redirecting')
data_json = self._search_json(
r'data\s*=', webpage, 'metadata', video_id, contains_pattern=r'{["\']_*serverState_*video.+}')
serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)', serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)',
webpage, 'server state').replace('State', 'Settings') webpage, 'server state').replace('State', 'Settings')
uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)', uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
@ -254,11 +277,12 @@ def _real_extract(self, url):
formats.extend(self._extract_m3u8_formats(s_url, id, 'mp4')) formats.extend(self._extract_m3u8_formats(s_url, id, 'mp4'))
self._sort_formats(formats) self._sort_formats(formats)
return { return {
'id': id, 'id': video_id,
'title': video_json.get('title') or self._og_search_title(webpage), 'title': video_json.get('title') or self._og_search_title(webpage),
'formats': formats, 'formats': formats,
'duration': int_or_none(video_json.get('duration')), 'duration': int_or_none(video_json.get('duration')),
'view_count': int_or_none(video_json.get('views')), 'view_count': int_or_none(video_json.get('views')),
'timestamp': int_or_none(video_json.get('publicationDate')),
'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']), 'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
'description': self._og_search_description(webpage) or try_get(data_json, lambda x: x['og']['description']), 'description': self._og_search_description(webpage) or try_get(data_json, lambda x: x['og']['description']),
'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']), 'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']),
@ -266,40 +290,99 @@ def _real_extract(self, url):
class ZenYandexChannelIE(InfoExtractor): class ZenYandexChannelIE(InfoExtractor):
_VALID_URL = r'https?://zen\.yandex\.ru/(?!media|video)(?:id/)?(?P<id>[a-z0-9-_]+)' _VALID_URL = r'https?://(zen\.yandex|dzen)\.ru/(?!media|video)(?:id/)?(?P<id>[a-z0-9-_]+)'
_TESTS = [{ _TESTS = [{
'url': 'https://zen.yandex.ru/tok_media', 'url': 'https://zen.yandex.ru/tok_media',
'info_dict': { 'info_dict': {
'id': 'tok_media', 'id': 'tok_media',
'title': 'СПЕКТР',
'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
},
'playlist_mincount': 169,
}, {
'url': 'https://dzen.ru/tok_media',
'info_dict': {
'id': 'tok_media',
'title': 'СПЕКТР',
'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
}, },
'playlist_mincount': 169, 'playlist_mincount': 169,
}, { }, {
'url': 'https://zen.yandex.ru/id/606fd806cc13cb3c58c05cf5', 'url': 'https://zen.yandex.ru/id/606fd806cc13cb3c58c05cf5',
'info_dict': { 'info_dict': {
'id': '606fd806cc13cb3c58c05cf5', 'id': '606fd806cc13cb3c58c05cf5',
'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
'title': 'AcademeG DailyStream',
},
'playlist_mincount': 657,
}, {
# Test that the playlist extractor finishes extracting when the
# channel has less than one page
'url': 'https://zen.yandex.ru/jony_me',
'info_dict': {
'id': 'jony_me',
'description': 'md5:a2c62b4ef5cf3e3efb13d25f61f739e1',
'title': 'JONY ',
},
'playlist_count': 20,
}, {
# Test that the playlist extractor finishes extracting when the
# channel has more than one page of entries
'url': 'https://zen.yandex.ru/tatyanareva',
'info_dict': {
'id': 'tatyanareva',
'description': 'md5:296b588d60841c3756c9105f237b70c6',
'title': 'Татьяна Рева',
'entries': 'maxcount:200',
},
'playlist_count': 46,
}, {
'url': 'https://dzen.ru/id/606fd806cc13cb3c58c05cf5',
'info_dict': {
'id': '606fd806cc13cb3c58c05cf5',
'title': 'AcademeG DailyStream',
'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
}, },
'playlist_mincount': 657, 'playlist_mincount': 657,
}] }]
def _entries(self, id, url): def _entries(self, item_id, server_state_json, server_settings_json):
webpage = self._download_webpage(url, id) items = (traverse_obj(server_state_json, ('feed', 'items', ...))
data_json = self._parse_json(re.findall(r'var\s?data\s?=\s?({.+?})\s?;', webpage)[-1], id) or traverse_obj(server_settings_json, ('exportData', 'items', ...)))
for key in data_json.keys():
if key.startswith('__serverState__'): more = (traverse_obj(server_state_json, ('links', 'more'))
data_json = data_json[key] or traverse_obj(server_settings_json, ('exportData', 'more', 'link')))
items = list(try_get(data_json, lambda x: x['feed']['items'], dict).values())
more = try_get(data_json, lambda x: x['links']['more']) or None next_page_id = None
for page in itertools.count(1): for page in itertools.count(1):
for item in items: for item in items or []:
video_id = item.get('publication_id') or item.get('publicationId') if item.get('type') != 'gif':
video_url = item.get('link') continue
yield self.url_result(video_url, ie=ZenYandexIE.ie_key(), video_id=video_id.split(':')[-1]) video_id = traverse_obj(item, 'publication_id', 'publicationId') or ''
if not more: yield self.url_result(item['link'], ZenYandexIE, video_id.split(':')[-1])
current_page_id = next_page_id
next_page_id = traverse_obj(parse_qs(more), ('next_page_id', -1))
if not all((more, items, next_page_id, next_page_id != current_page_id)):
break break
data_json = self._download_json(more, id, note='Downloading Page %d' % page)
items = data_json.get('items', []) data = self._download_json(more, item_id, note=f'Downloading Page {page}')
more = try_get(data_json, lambda x: x['more']['link']) or None items, more = data.get('items'), traverse_obj(data, ('more', 'link'))
def _real_extract(self, url): def _real_extract(self, url):
id = self._match_id(url) item_id = self._match_id(url)
return self.playlist_result(self._entries(id, url), playlist_id=id) webpage = self._download_webpage(url, item_id)
redirect = self._search_json(
r'var it\s*=\s*', webpage, 'redirect', item_id, default={}).get('retpath')
if redirect:
item_id = self._match_id(redirect)
webpage = self._download_webpage(redirect, item_id, note='Redirecting')
data = self._search_json(
r'var\s+data\s*=', webpage, 'channel data', item_id, contains_pattern=r'{\"__serverState__.+}')
server_state_json = traverse_obj(data, lambda k, _: k.startswith('__serverState__'), get_all=False)
server_settings_json = traverse_obj(data, lambda k, _: k.startswith('__serverSettings__'), get_all=False)
return self.playlist_result(
self._entries(item_id, server_state_json, server_settings_json),
item_id, traverse_obj(server_state_json, ('channel', 'source', 'title')),
traverse_obj(server_state_json, ('channel', 'source', 'description')))