mirror of https://github.com/yt-dlp/yt-dlp.git
Compare commits
10 Commits
6aee2250ce
...
7f5b0df8e1
Author | SHA1 | Date |
---|---|---|
lauren n. liberda | 7f5b0df8e1 | |
bashonly | 89f535e265 | |
bashonly | ff38a011d5 | |
bashonly | 8056a3026e | |
Simon Sawicki | 3ee1194288 | |
Lauren N. Liberda | 2a298347e1 | |
Lauren N. Liberda | 7a1f3a1c70 | |
Lauren N. Liberda | f4ba2275f2 | |
Lauren N. Liberda | 9d1c75df0d | |
Lauren N. Liberda | d44e5195cb |
|
@ -53,7 +53,7 @@ jobs:
|
|||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install test requirements
|
||||
run: python3 ./devscripts/install_deps.py --include dev --include curl_cffi
|
||||
run: python3 ./devscripts/install_deps.py --include dev --include curl-cffi
|
||||
- name: Run tests
|
||||
continue-on-error: False
|
||||
run: |
|
||||
|
|
|
@ -1906,6 +1906,15 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
|
|||
expected_status=TEAPOT_RESPONSE_STATUS)
|
||||
self.assertEqual(content, TEAPOT_RESPONSE_BODY)
|
||||
|
||||
def test_search_nextjs_data(self):
|
||||
data = '<script id="__NEXT_DATA__" type="application/json">{"props":{}}</script>'
|
||||
self.assertEqual(self.ie._search_nextjs_data(data, None), {'props': {}})
|
||||
self.assertEqual(self.ie._search_nextjs_data('', None, fatal=False), {})
|
||||
self.assertEqual(self.ie._search_nextjs_data('', None, default=None), None)
|
||||
self.assertEqual(self.ie._search_nextjs_data('', None, default={}), {})
|
||||
with self.assertRaises(DeprecationWarning):
|
||||
self.assertEqual(self.ie._search_nextjs_data('', None, default='{}'), {})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
@ -105,7 +105,7 @@ class AsobiStageIE(InfoExtractor):
|
|||
video_type = {'archive': 'archives', 'player': 'broadcasts'}[type_]
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
event_data = traverse_obj(
|
||||
self._search_nextjs_data(webpage, video_id, default='{}'),
|
||||
self._search_nextjs_data(webpage, video_id, default={}),
|
||||
('props', 'pageProps', 'eventCMSData', {
|
||||
'title': ('event_name', {str}),
|
||||
'thumbnail': ('event_thumbnail_image', {url_or_none}),
|
||||
|
|
|
@ -1738,12 +1738,16 @@ class InfoExtractor:
|
|||
traverse_json_ld(json_ld)
|
||||
return filter_dict(info)
|
||||
|
||||
def _search_nextjs_data(self, webpage, video_id, *, transform_source=None, fatal=True, **kw):
|
||||
return self._parse_json(
|
||||
self._search_regex(
|
||||
r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
|
||||
webpage, 'next.js data', fatal=fatal, **kw),
|
||||
video_id, transform_source=transform_source, fatal=fatal)
|
||||
def _search_nextjs_data(self, webpage, video_id, *, fatal=True, default=NO_DEFAULT, **kw):
|
||||
if default == '{}':
|
||||
self._downloader.deprecation_warning('using `default=\'{}\'` is deprecated, use `default={}` instead')
|
||||
default = {}
|
||||
if default is not NO_DEFAULT:
|
||||
fatal = False
|
||||
|
||||
return self._search_json(
|
||||
r'<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>', webpage, 'next.js data',
|
||||
video_id, end_pattern='</script>', fatal=fatal, default=default, **kw)
|
||||
|
||||
def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', *, fatal=True, traverse=('data', 0)):
|
||||
"""Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
|
||||
|
|
|
@ -24,11 +24,15 @@ class CrunchyrollBaseIE(InfoExtractor):
|
|||
_BASE_URL = 'https://www.crunchyroll.com'
|
||||
_API_BASE = 'https://api.crunchyroll.com'
|
||||
_NETRC_MACHINE = 'crunchyroll'
|
||||
_REFRESH_TOKEN = None
|
||||
_AUTH_HEADERS = None
|
||||
_AUTH_EXPIRY = None
|
||||
_API_ENDPOINT = None
|
||||
_BASIC_AUTH = None
|
||||
_BASIC_AUTH = 'Basic ' + base64.b64encode(':'.join((
|
||||
't-kdgp2h8c3jub8fn0fq',
|
||||
'yfLDfMfrYvKXh4JXS1LEI2cCqu1v5Wan',
|
||||
)).encode()).decode()
|
||||
_IS_PREMIUM = None
|
||||
_CLIENT_ID = ('cr_web', 'noaihdevm_6iyg0a8l0q')
|
||||
_LOCALE_LOOKUP = {
|
||||
'ar': 'ar-SA',
|
||||
'de': 'de-DE',
|
||||
|
@ -43,69 +47,74 @@ class CrunchyrollBaseIE(InfoExtractor):
|
|||
'hi': 'hi-IN',
|
||||
}
|
||||
|
||||
@property
|
||||
def is_logged_in(self):
|
||||
return bool(self._get_cookies(self._BASE_URL).get('etp_rt'))
|
||||
def _set_auth_info(self, response):
|
||||
CrunchyrollBaseIE._IS_PREMIUM = 'cr_premium' in traverse_obj(response, ('access_token', {jwt_decode_hs256}, 'benefits', ...))
|
||||
CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': response['token_type'] + ' ' + response['access_token']}
|
||||
CrunchyrollBaseIE._AUTH_EXPIRY = time_seconds(seconds=traverse_obj(response, ('expires_in', {float_or_none}), default=300) - 10)
|
||||
|
||||
def _request_token(self, headers, data, note='Requesting token', errnote='Failed to request token'):
|
||||
try: # TODO: Add impersonation support here
|
||||
return self._download_json(
|
||||
f'{self._BASE_URL}/auth/v1/token', None, note=note, errnote=errnote,
|
||||
headers=headers, data=urlencode_postdata(data))
|
||||
except ExtractorError as error:
|
||||
if not isinstance(error.cause, HTTPError) or error.cause.status != 403:
|
||||
raise
|
||||
raise ExtractorError(
|
||||
'Request blocked by Cloudflare; navigate to Crunchyroll in your browser, '
|
||||
'then pass the fresh cookies (with --cookies-from-browser or --cookies) '
|
||||
'and your browser\'s User-Agent (with --user-agent)', expected=True)
|
||||
|
||||
def _perform_login(self, username, password):
|
||||
if self.is_logged_in:
|
||||
if not CrunchyrollBaseIE._REFRESH_TOKEN:
|
||||
CrunchyrollBaseIE._REFRESH_TOKEN = self.cache.load(self._NETRC_MACHINE, username)
|
||||
if CrunchyrollBaseIE._REFRESH_TOKEN:
|
||||
return
|
||||
|
||||
upsell_response = self._download_json(
|
||||
f'{self._API_BASE}/get_upsell_data.0.json', None, 'Getting session id',
|
||||
query={
|
||||
'sess_id': 1,
|
||||
'device_id': 'whatvalueshouldbeforweb',
|
||||
'device_type': 'com.crunchyroll.static',
|
||||
'access_token': 'giKq5eY27ny3cqz',
|
||||
'referer': f'{self._BASE_URL}/welcome/login'
|
||||
})
|
||||
if upsell_response['code'] != 'ok':
|
||||
raise ExtractorError('Could not get session id')
|
||||
session_id = upsell_response['data']['session_id']
|
||||
|
||||
login_response = self._download_json(
|
||||
f'{self._API_BASE}/login.1.json', None, 'Logging in',
|
||||
data=urlencode_postdata({
|
||||
'account': username,
|
||||
'password': password,
|
||||
'session_id': session_id
|
||||
}))
|
||||
if login_response['code'] != 'ok':
|
||||
raise ExtractorError('Login failed. Server message: %s' % login_response['message'], expected=True)
|
||||
if not self.is_logged_in:
|
||||
raise ExtractorError('Login succeeded but did not set etp_rt cookie')
|
||||
|
||||
def _update_auth(self):
|
||||
if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_REFRESH > time_seconds():
|
||||
return
|
||||
|
||||
if not CrunchyrollBaseIE._BASIC_AUTH:
|
||||
cx_api_param = self._CLIENT_ID[self.is_logged_in]
|
||||
self.write_debug(f'Using cxApiParam={cx_api_param}')
|
||||
CrunchyrollBaseIE._BASIC_AUTH = 'Basic ' + base64.b64encode(f'{cx_api_param}:'.encode()).decode()
|
||||
|
||||
auth_headers = {'Authorization': CrunchyrollBaseIE._BASIC_AUTH}
|
||||
if self.is_logged_in:
|
||||
grant_type = 'etp_rt_cookie'
|
||||
else:
|
||||
grant_type = 'client_id'
|
||||
auth_headers['ETP-Anonymous-ID'] = uuid.uuid4()
|
||||
try:
|
||||
auth_response = self._download_json(
|
||||
f'{self._BASE_URL}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
|
||||
headers=auth_headers, data=f'grant_type={grant_type}'.encode())
|
||||
login_response = self._request_token(
|
||||
headers={'Authorization': self._BASIC_AUTH}, data={
|
||||
'username': username,
|
||||
'password': password,
|
||||
'grant_type': 'password',
|
||||
'scope': 'offline_access',
|
||||
}, note='Logging in', errnote='Failed to log in')
|
||||
except ExtractorError as error:
|
||||
if isinstance(error.cause, HTTPError) and error.cause.status == 403:
|
||||
raise ExtractorError(
|
||||
'Request blocked by Cloudflare; navigate to Crunchyroll in your browser, '
|
||||
'then pass the fresh cookies (with --cookies-from-browser or --cookies) '
|
||||
'and your browser\'s User-Agent (with --user-agent)', expected=True)
|
||||
if isinstance(error.cause, HTTPError) and error.cause.status == 401:
|
||||
raise ExtractorError('Invalid username and/or password', expected=True)
|
||||
raise
|
||||
|
||||
CrunchyrollBaseIE._IS_PREMIUM = 'cr_premium' in traverse_obj(auth_response, ('access_token', {jwt_decode_hs256}, 'benefits', ...))
|
||||
CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']}
|
||||
CrunchyrollBaseIE._AUTH_REFRESH = time_seconds(seconds=traverse_obj(auth_response, ('expires_in', {float_or_none}), default=300) - 10)
|
||||
CrunchyrollBaseIE._REFRESH_TOKEN = login_response['refresh_token']
|
||||
self.cache.store(self._NETRC_MACHINE, username, CrunchyrollBaseIE._REFRESH_TOKEN)
|
||||
self._set_auth_info(login_response)
|
||||
|
||||
def _update_auth(self):
|
||||
if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_EXPIRY > time_seconds():
|
||||
return
|
||||
|
||||
auth_headers = {'Authorization': self._BASIC_AUTH}
|
||||
if CrunchyrollBaseIE._REFRESH_TOKEN:
|
||||
data = {
|
||||
'refresh_token': CrunchyrollBaseIE._REFRESH_TOKEN,
|
||||
'grant_type': 'refresh_token',
|
||||
'scope': 'offline_access',
|
||||
}
|
||||
else:
|
||||
data = {'grant_type': 'client_id'}
|
||||
auth_headers['ETP-Anonymous-ID'] = uuid.uuid4()
|
||||
try:
|
||||
auth_response = self._request_token(auth_headers, data)
|
||||
except ExtractorError as error:
|
||||
username, password = self._get_login_info()
|
||||
if not username or not isinstance(error.cause, HTTPError) or error.cause.status != 400:
|
||||
raise
|
||||
self.to_screen('Refresh token has expired. Re-logging in')
|
||||
CrunchyrollBaseIE._REFRESH_TOKEN = None
|
||||
self.cache.store(self._NETRC_MACHINE, username, None)
|
||||
self._perform_login(username, password)
|
||||
return
|
||||
|
||||
self._set_auth_info(auth_response)
|
||||
|
||||
def _locale_from_language(self, language):
|
||||
config_locale = self._configuration_arg('metadata', ie_key=CrunchyrollBetaIE, casesense=True)
|
||||
|
@ -168,7 +177,8 @@ class CrunchyrollBaseIE(InfoExtractor):
|
|||
self._update_auth()
|
||||
stream_response = self._download_json(
|
||||
f'https://cr-play-service.prd.crunchyrollsvc.com/v1/{identifier}/console/switch/play',
|
||||
display_id, note='Downloading stream info', headers=CrunchyrollBaseIE._AUTH_HEADERS)
|
||||
display_id, note='Downloading stream info', errnote='Failed to download stream info',
|
||||
headers=CrunchyrollBaseIE._AUTH_HEADERS)
|
||||
|
||||
available_formats = {'': ('', '', stream_response['url'])}
|
||||
for hardsub_lang, stream in traverse_obj(stream_response, ('hardSubs', {dict.items}, lambda _, v: v[1]['url'])):
|
||||
|
@ -383,9 +393,9 @@ class CrunchyrollBetaIE(CrunchyrollCmsBaseIE):
|
|||
|
||||
if not self._IS_PREMIUM and traverse_obj(response, (f'{object_type}_metadata', 'is_premium_only')):
|
||||
message = f'This {object_type} is for premium members only'
|
||||
if self.is_logged_in:
|
||||
if CrunchyrollBaseIE._REFRESH_TOKEN:
|
||||
raise ExtractorError(message, expected=True)
|
||||
self.raise_login_required(message)
|
||||
self.raise_login_required(message, method='password')
|
||||
|
||||
result['formats'], result['subtitles'] = self._extract_stream(internal_id)
|
||||
|
||||
|
@ -575,9 +585,9 @@ class CrunchyrollMusicIE(CrunchyrollBaseIE):
|
|||
|
||||
if not self._IS_PREMIUM and response.get('isPremiumOnly'):
|
||||
message = f'This {response.get("type") or "media"} is for premium members only'
|
||||
if self.is_logged_in:
|
||||
if CrunchyrollBaseIE._REFRESH_TOKEN:
|
||||
raise ExtractorError(message, expected=True)
|
||||
self.raise_login_required(message)
|
||||
self.raise_login_required(message, method='password')
|
||||
|
||||
result = self._transform_music_response(response)
|
||||
result['formats'], _ = self._extract_stream(f'music/{internal_id}', internal_id)
|
||||
|
|
|
@ -174,7 +174,7 @@ class TheaterComplexTownBaseIE(StacommuBaseIE):
|
|||
|
||||
|
||||
class TheaterComplexTownVODIE(TheaterComplexTownBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?theater-complex\.town/(?:en/)?videos/episodes/(?P<id>\w+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?theater-complex\.town/(?:(?:en|ja)/)?videos/episodes/(?P<id>\w+)'
|
||||
IE_NAME = 'theatercomplextown:vod'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.theater-complex.town/videos/episodes/hoxqidYNoAn7bP92DN6p78',
|
||||
|
@ -195,6 +195,9 @@ class TheaterComplexTownVODIE(TheaterComplexTownBaseIE):
|
|||
}, {
|
||||
'url': 'https://www.theater-complex.town/en/videos/episodes/6QT7XYwM9dJz5Gf9VB6K5y',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.theater-complex.town/ja/videos/episodes/hoxqidYNoAn7bP92DN6p78',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_API_PATH = 'videoEpisodes'
|
||||
|
@ -204,7 +207,7 @@ class TheaterComplexTownVODIE(TheaterComplexTownBaseIE):
|
|||
|
||||
|
||||
class TheaterComplexTownPPVIE(TheaterComplexTownBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?theater-complex\.town/(?:en/)?ppv/(?P<id>\w+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?theater-complex\.town/(?:(?:en|ja)/)?ppv/(?P<id>\w+)'
|
||||
IE_NAME = 'theatercomplextown:ppv'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.theater-complex.town/ppv/wytW3X7khrjJBUpKuV3jen',
|
||||
|
@ -223,6 +226,9 @@ class TheaterComplexTownPPVIE(TheaterComplexTownBaseIE):
|
|||
}, {
|
||||
'url': 'https://www.theater-complex.town/en/ppv/wytW3X7khrjJBUpKuV3jen',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.theater-complex.town/ja/ppv/qwUVmLmGEiZ3ZW6it9uGys',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_API_PATH = 'events'
|
||||
|
|
|
@ -41,7 +41,7 @@ class STVPlayerIE(InfoExtractor):
|
|||
ptype, video_id = self._match_valid_url(url).groups()
|
||||
|
||||
webpage = self._download_webpage(url, video_id, fatal=False) or ''
|
||||
props = self._search_nextjs_data(webpage, video_id, default='{}').get('props') or {}
|
||||
props = self._search_nextjs_data(webpage, video_id, default={}).get('props') or {}
|
||||
player_api_cache = try_get(
|
||||
props, lambda x: x['initialReduxState']['playerApiCache']) or {}
|
||||
|
||||
|
|
|
@ -776,7 +776,7 @@ class TikTokIE(TikTokBaseIE):
|
|||
status = traverse_obj(sigi_data, ('VideoPage', 'statusCode', {int})) or 0
|
||||
video_data = traverse_obj(sigi_data, ('ItemModule', video_id, {dict}))
|
||||
|
||||
elif next_data := self._search_nextjs_data(webpage, video_id, default='{}'):
|
||||
elif next_data := self._search_nextjs_data(webpage, video_id, default={}):
|
||||
self.write_debug('Found next.js data')
|
||||
status = traverse_obj(next_data, ('props', 'pageProps', 'statusCode', {int})) or 0
|
||||
video_data = traverse_obj(next_data, ('props', 'pageProps', 'itemInfo', 'itemStruct', {dict}))
|
||||
|
|
|
@ -8,7 +8,7 @@ from ..utils import (
|
|||
|
||||
|
||||
class TumblrIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?P<blog_name>[^/?#&]+)\.tumblr\.com/(?:post|video)/(?P<id>[0-9]+)(?:$|[/?#])'
|
||||
_VALID_URL = r'https?://(?P<blog_name_1>[^/?#&]+)\.tumblr\.com/(?:post|video|(?P<blog_name_2>[a-zA-Z\d-]+))/(?P<id>[0-9]+)(?:$|[/?#])'
|
||||
_NETRC_MACHINE = 'tumblr'
|
||||
_LOGIN_URL = 'https://www.tumblr.com/login'
|
||||
_OAUTH_URL = 'https://www.tumblr.com/api/v2/oauth2/token'
|
||||
|
@ -109,14 +109,13 @@ class TumblrIE(InfoExtractor):
|
|||
'title': 'The Blues Remembers Everything the Country Forgot',
|
||||
'alt_title': 'The Blues Remembers Everything the Country Forgot',
|
||||
'description': 'md5:1a6b4097e451216835a24c1023707c79',
|
||||
'release_date': '20201224',
|
||||
'creator': 'md5:c2239ba15430e87c3b971ba450773272',
|
||||
'uploader': 'Moor Mother - Topic',
|
||||
'upload_date': '20201223',
|
||||
'uploader_id': 'UCxrMtFBRkFvQJ_vVM4il08w',
|
||||
'uploader_url': 'http://www.youtube.com/channel/UCxrMtFBRkFvQJ_vVM4il08w',
|
||||
'thumbnail': r're:^https?://i.ytimg.com/.*',
|
||||
'channel': 'Moor Mother - Topic',
|
||||
'channel': 'Moor Mother',
|
||||
'channel_id': 'UCxrMtFBRkFvQJ_vVM4il08w',
|
||||
'channel_url': 'https://www.youtube.com/channel/UCxrMtFBRkFvQJ_vVM4il08w',
|
||||
'channel_follower_count': int,
|
||||
|
@ -135,21 +134,6 @@ class TumblrIE(InfoExtractor):
|
|||
'release_year': 2020,
|
||||
},
|
||||
'add_ie': ['Youtube'],
|
||||
}, {
|
||||
'url': 'http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching',
|
||||
'md5': 'de07e5211d60d4f3a2c3df757ea9f6ab',
|
||||
'info_dict': {
|
||||
'id': 'Wmur',
|
||||
'ext': 'mp4',
|
||||
'title': 'naked smoking & stretching',
|
||||
'upload_date': '20150506',
|
||||
'timestamp': 1430931613,
|
||||
'age_limit': 18,
|
||||
'uploader_id': '1638622',
|
||||
'uploader': 'naked-yogi',
|
||||
},
|
||||
# 'add_ie': ['Vidme'],
|
||||
'skip': 'dead embedded video host'
|
||||
}, {
|
||||
'url': 'https://prozdvoices.tumblr.com/post/673201091169681408/what-recording-voice-acting-sounds-like',
|
||||
'md5': 'a0063fc8110e6c9afe44065b4ea68177',
|
||||
|
@ -232,6 +216,139 @@ class TumblrIE(InfoExtractor):
|
|||
'upload_date': '20140429',
|
||||
},
|
||||
'add_ie': ['Instagram'],
|
||||
}, {
|
||||
'note': 'new url scheme',
|
||||
'url': 'https://www.tumblr.com/catgirldick/706354197596078080?source=share',
|
||||
'info_dict': {
|
||||
'id': '706354197596078080',
|
||||
'ext': 'mp4',
|
||||
'title': 'Bocchi in low quality and spinning, nothing just that',
|
||||
'description': 'Bocchi in low quality and spinning, nothing just that',
|
||||
'tags': [],
|
||||
'uploader_id': str,
|
||||
'uploader_url': r're:https://[^/]+\.tumblr\.com/',
|
||||
'thumbnail': r're:https?://[^/]+\.media\.tumblr\.com/[^?#]+.jpg',
|
||||
'repost_count': int,
|
||||
'like_count': int,
|
||||
'age_limit': 0,
|
||||
},
|
||||
}, {
|
||||
'note': 'bandcamp album embed',
|
||||
'url': 'https://patricia-taxxon.tumblr.com/post/704473755725004800/patricia-taxxon-agnes-hilda-patricia-taxxon',
|
||||
'info_dict': {
|
||||
'id': 'agnes-hilda',
|
||||
'title': 'Agnes & Hilda',
|
||||
'description': 'The inexplicable joy of an artist. Wash paws after listening.',
|
||||
'uploader_id': 'patriciataxxon',
|
||||
},
|
||||
'playlist_count': 8,
|
||||
}, {
|
||||
'note': 'bandcamp track embeds (many)',
|
||||
'url': 'https://lyuboserafimov.tumblr.com/post/706659328557514752/dream-sequencer-i-love-you-sarah-connor-dream',
|
||||
'info_dict': {
|
||||
'id': '706659328557514752',
|
||||
'title': 'md5:d20e162d74d4225ef19ef5700760ea4e',
|
||||
'description': 'md5:ede184a5af7e79f09e5835d65986dd1f',
|
||||
'tags': ['synthwave', 'dream sequencer', 'retrowave', '80s', 'nostalgia', 'Bandcamp', 'My Music'],
|
||||
'uploader_id': 'lyuboserafimov',
|
||||
'uploader_url': 'https://lyuboserafimov.tumblr.com/',
|
||||
'age_limit': 0,
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
},
|
||||
'playlist_count': 4,
|
||||
}, {
|
||||
'note': 'soundcloud track embed',
|
||||
'url': 'https://selfisekai.tumblr.com/post/706878583155671040/dont-mind-me-im-just-a-lil-yt-dlp-contributor',
|
||||
'info_dict': {
|
||||
'id': '1413238837',
|
||||
'ext': 'mp3',
|
||||
'title': '100 gecs - 800db cloud (maiacore nightcore edit)',
|
||||
'description': '',
|
||||
'genre': 'nightcore',
|
||||
'license': 'all-rights-reserved',
|
||||
'uploader': 'maiacore',
|
||||
'uploader_id': '1109090314',
|
||||
'uploader_url': 'https://soundcloud.com/maiacore',
|
||||
'duration': 97.045,
|
||||
'timestamp': 1672430792,
|
||||
'upload_date': '20221230',
|
||||
'thumbnail': r're:https?://.+\.jpg',
|
||||
'view_count': int,
|
||||
'repost_count': int,
|
||||
'like_count': int,
|
||||
'comment_count': int,
|
||||
},
|
||||
}, {
|
||||
'note': 'soundcloud set embed',
|
||||
'url': 'https://selfisekai.tumblr.com/post/706885093077237760/songs-with-maia-in-them',
|
||||
'info_dict': {
|
||||
'id': '1369208083',
|
||||
'title': 'songs with maia in them :)',
|
||||
'description': '',
|
||||
},
|
||||
'playlist_mincount': 8,
|
||||
}, {
|
||||
'note': 'dailymotion video embed',
|
||||
'url': 'https://www.tumblr.com/selfisekai/706884794734313472?source=share',
|
||||
'info_dict': {
|
||||
'id': 'x8hczf5',
|
||||
'ext': 'mp4',
|
||||
'title': 'Trying a viral pizza bagel in NYC',
|
||||
'description': 'md5:17e52b32ae21f23940912b3efff17b74',
|
||||
'duration': 58,
|
||||
'uploader': 'Insider',
|
||||
'uploader_id': 'x29n239',
|
||||
'tags': ['food', 'video', 'pizza', 'foody', 'bagel', 'bagels', 'feed:ins', 'feed:live', 'jacky barile', 'reels'],
|
||||
'timestamp': 1674064861,
|
||||
'upload_date': '20230118',
|
||||
'age_limit': 0,
|
||||
'thumbnail': 're:https?://.+',
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
},
|
||||
}, {
|
||||
'note': 'tiktok video embed',
|
||||
'url': 'https://selfisekai.tumblr.com/post/706885498468270080',
|
||||
'info_dict': {
|
||||
'id': '7098761136534867205',
|
||||
'ext': 'mp4',
|
||||
'title': 'md5:6e62de3b0157d2ed9999c7d61c6485f2',
|
||||
'description': 'md5:6e62de3b0157d2ed9999c7d61c6485f2',
|
||||
'creator': 'Poznań 🐐',
|
||||
'uploader': 'miasto_poznan',
|
||||
'uploader_id': '6998015174997607430',
|
||||
'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAzgdq4BFpzWfUWq4sO84_HGHHdS7cItoTDdAscuSxYIUQKJvZZ9j99wPe0RuqJpaR',
|
||||
'duration': 46,
|
||||
'timestamp': 1652809127,
|
||||
'upload_date': '20220517',
|
||||
'thumbnail': r're:https?://[^/]+\.tiktokcdn\.com/.+',
|
||||
'artist': 'The King Khan & BBQ Show',
|
||||
'album': 'Love You So',
|
||||
'track': 'Love You So',
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'comment_count': int,
|
||||
'repost_count': int,
|
||||
},
|
||||
}, {
|
||||
'note': 'tumblr video AND youtube embed',
|
||||
'url': 'https://selfisekai.tumblr.com/post/706895394042511360',
|
||||
'info_dict': {
|
||||
'id': '706895394042511360',
|
||||
'title': str,
|
||||
'uploader_id': 'selfisekai',
|
||||
'uploader_url': 'https://selfisekai.tumblr.com/',
|
||||
'age_limit': 0,
|
||||
'tags': [],
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
},
|
||||
'playlist_count': 2,
|
||||
}, {
|
||||
# twitch_live provider - error when linked account is not live
|
||||
'url': 'https://www.tumblr.com/anarcho-skamunist/722224493650722816/hollow-knight-stream-right-now-going-to-fight',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_providers = {
|
||||
|
@ -239,6 +356,20 @@ class TumblrIE(InfoExtractor):
|
|||
'vimeo': 'Vimeo',
|
||||
'vine': 'Vine',
|
||||
'youtube': 'Youtube',
|
||||
'dailymotion': 'Dailymotion',
|
||||
'tiktok': 'TikTok',
|
||||
'twitch_live': 'TwitchStream',
|
||||
}
|
||||
# these are known providers, but we don't know which entity type
|
||||
# we are supposed to extract, so we use matching by url
|
||||
_ambiguous_providers = {
|
||||
'bandcamp',
|
||||
'soundcloud',
|
||||
}
|
||||
# known not to be supported
|
||||
_unsupported_providers = {
|
||||
# seems like podcasts can't be embedded
|
||||
'spotify',
|
||||
}
|
||||
|
||||
_ACCESS_TOKEN = None
|
||||
|
@ -256,23 +387,41 @@ class TumblrIE(InfoExtractor):
|
|||
if not self._ACCESS_TOKEN:
|
||||
return
|
||||
|
||||
self._download_json(
|
||||
self._OAUTH_URL, None, 'Logging in',
|
||||
data=urlencode_postdata({
|
||||
'password': password,
|
||||
'grant_type': 'password',
|
||||
'username': username,
|
||||
}), headers={
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'Authorization': f'Bearer {self._ACCESS_TOKEN}',
|
||||
},
|
||||
errnote='Login failed', fatal=False)
|
||||
data = {
|
||||
'password': password,
|
||||
'grant_type': 'password',
|
||||
'username': username,
|
||||
}
|
||||
if self.get_param('twofactor'):
|
||||
data['tfa_token'] = self.get_param('twofactor')
|
||||
|
||||
def _call_login():
|
||||
return self._download_json(
|
||||
self._OAUTH_URL, None, 'Logging in',
|
||||
data=urlencode_postdata(data),
|
||||
headers={
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'Authorization': f'Bearer {self._ACCESS_TOKEN}',
|
||||
},
|
||||
errnote='Login failed', fatal=False,
|
||||
expected_status=lambda s: s == 200 or 400 <= s < 500)
|
||||
|
||||
response = _call_login()
|
||||
if response.get('error') == 'tfa_required':
|
||||
data['tfa_token'] = self._get_tfa_info()
|
||||
response = _call_login()
|
||||
if response.get('error'):
|
||||
self.report_warning('API returned error {}: {}'.format(
|
||||
response.get('error'), response.get('error_description')))
|
||||
|
||||
def _real_extract(self, url):
|
||||
blog, video_id = self._match_valid_url(url).groups()
|
||||
blog_1, blog_2, video_id = self._match_valid_url(url).groups()
|
||||
blog = blog_2 or blog_1
|
||||
|
||||
url = f'http://{blog}.tumblr.com/post/{video_id}/'
|
||||
webpage, urlh = self._download_webpage_handle(url, video_id)
|
||||
url = f'http://{blog}.tumblr.com/post/{video_id}'
|
||||
# whatsapp ua makes iab tcf shut the fuck up
|
||||
webpage, urlh = self._download_webpage_handle(url, video_id, headers={
|
||||
'User-Agent': 'WhatsApp/2.0'})
|
||||
|
||||
redirect_url = urlh.url
|
||||
|
||||
|
@ -291,21 +440,70 @@ class TumblrIE(InfoExtractor):
|
|||
video_id, headers={'Authorization': f'Bearer {self._ACCESS_TOKEN}'}, fatal=False),
|
||||
('response', 'timeline', 'elements', 0)) or {}
|
||||
content_json = traverse_obj(post_json, ('trail', 0, 'content'), ('content')) or []
|
||||
video_json = next(
|
||||
(item for item in content_json if item.get('type') == 'video'), {})
|
||||
media_json = video_json.get('media') or {}
|
||||
if api_only and not media_json.get('url') and not video_json.get('url'):
|
||||
raise ExtractorError('Failed to find video data for dashboard-only post')
|
||||
|
||||
if not media_json.get('url') and video_json.get('url'):
|
||||
# external video host
|
||||
return self.url_result(
|
||||
video_json['url'],
|
||||
self._providers.get(video_json.get('provider'), 'Generic'))
|
||||
# the url we're extracting from might be an original post or it might be a reblog.
|
||||
# if it's a reblog, og:description will be the reblogger's comment, not the uploader's.
|
||||
# content_json is always the op, so if it exists but has no text, there's no description
|
||||
if content_json:
|
||||
description = '\n\n'.join((
|
||||
item.get('text') for item in content_json if item.get('type') == 'text')) or None
|
||||
else:
|
||||
description = self._og_search_description(webpage, default=None)
|
||||
uploader_id = traverse_obj(post_json, 'reblogged_root_name', 'blog_name')
|
||||
|
||||
video_url = self._og_search_video_url(webpage, default=None)
|
||||
duration = None
|
||||
info_dict = {
|
||||
'id': video_id,
|
||||
'title': post_json.get('summary') or (blog if api_only else self._html_search_regex(
|
||||
r'(?s)<title>(?P<title>.*?)(?: \| Tumblr)?</title>', webpage, 'title', default=blog)),
|
||||
'description': description,
|
||||
'uploader_id': uploader_id,
|
||||
'uploader_url': f'https://{uploader_id}.tumblr.com/' if uploader_id else None,
|
||||
'like_count': post_json.get('like_count'),
|
||||
'repost_count': post_json.get('reblog_count'),
|
||||
'age_limit': {True: 18, False: 0}.get(post_json.get('is_nsfw')),
|
||||
'tags': post_json.get('tags'),
|
||||
}
|
||||
|
||||
# for tumblr's own video hosting
|
||||
fallback_format = None
|
||||
formats = []
|
||||
video_url = self._og_search_video_url(webpage, default=None)
|
||||
# for external video hosts
|
||||
entries = []
|
||||
ignored_providers = set()
|
||||
unknown_providers = set()
|
||||
|
||||
video_jsons = (item for item in content_json if item.get('type') in ('video', 'audio'))
|
||||
if video_jsons:
|
||||
for video_json in video_jsons:
|
||||
media_json = video_json.get('media') or {}
|
||||
if api_only and not media_json.get('url') and not video_json.get('url'):
|
||||
raise ExtractorError('Failed to find video data for dashboard-only post')
|
||||
provider = video_json.get('provider')
|
||||
is_provider_ambiguous = provider in self._ambiguous_providers
|
||||
|
||||
if provider in ('tumblr', None):
|
||||
fallback_format = {
|
||||
'url': media_json.get('url') or video_url,
|
||||
'width': int_or_none(
|
||||
media_json.get('width') or self._og_search_property('video:width', webpage, default=None)),
|
||||
'height': int_or_none(
|
||||
media_json.get('height') or self._og_search_property('video:height', webpage, default=None)),
|
||||
}
|
||||
continue
|
||||
elif provider in self._unsupported_providers:
|
||||
ignored_providers.add(provider)
|
||||
continue
|
||||
elif provider and not is_provider_ambiguous and provider not in self._providers:
|
||||
unknown_providers.add(provider)
|
||||
|
||||
if video_json.get('url'):
|
||||
# external video host
|
||||
entries.append(self.url_result(
|
||||
video_json['url'],
|
||||
self._providers.get(provider, 'Generic') if not is_provider_ambiguous else None))
|
||||
|
||||
duration = None
|
||||
|
||||
# iframes can supply duration and sometimes additional formats, so check for one
|
||||
iframe_url = self._search_regex(
|
||||
|
@ -344,44 +542,35 @@ class TumblrIE(InfoExtractor):
|
|||
'quality': quality,
|
||||
} for quality, (video_url, format_id) in enumerate(sources)]
|
||||
|
||||
if not media_json.get('url') and not video_url and not iframe_url:
|
||||
# external video host (but we weren't able to figure it out from the api)
|
||||
iframe_url = self._search_regex(
|
||||
r'src=["\'](https?://safe\.txmblr\.com/svc/embed/inline/[^"\']+)["\']',
|
||||
webpage, 'embed iframe url', default=None)
|
||||
return self.url_result(iframe_url or redirect_url, 'Generic')
|
||||
if not formats and fallback_format:
|
||||
formats.append(fallback_format)
|
||||
|
||||
formats = formats or [{
|
||||
'url': media_json.get('url') or video_url,
|
||||
'width': int_or_none(
|
||||
media_json.get('width') or self._og_search_property('video:width', webpage, default=None)),
|
||||
'height': int_or_none(
|
||||
media_json.get('height') or self._og_search_property('video:height', webpage, default=None)),
|
||||
}]
|
||||
if formats:
|
||||
# tumblr's own video is always above embeds
|
||||
entries = [{
|
||||
**info_dict,
|
||||
'formats': formats,
|
||||
'duration': duration,
|
||||
'thumbnail': (traverse_obj(video_json, ('poster', 0, 'url'))
|
||||
or self._og_search_thumbnail(webpage, default=None)),
|
||||
}] + entries
|
||||
|
||||
# the url we're extracting from might be an original post or it might be a reblog.
|
||||
# if it's a reblog, og:description will be the reblogger's comment, not the uploader's.
|
||||
# content_json is always the op, so if it exists but has no text, there's no description
|
||||
if content_json:
|
||||
description = '\n\n'.join((
|
||||
item.get('text') for item in content_json if item.get('type') == 'text')) or None
|
||||
if ignored_providers:
|
||||
if not entries:
|
||||
raise ExtractorError(f'None of embed providers are supported: {str(", ".join(ignored_providers))}', video_id=video_id, expected=True)
|
||||
else:
|
||||
self.report_warning(f'Embeds from these providers are ignored as unsupported: {str(", ".join(ignored_providers))}', video_id)
|
||||
if unknown_providers:
|
||||
self.report_warning(f'Unrecognized providers, please report: {str(", ".join(unknown_providers))}', video_id)
|
||||
|
||||
if len(entries) > 1:
|
||||
return {
|
||||
**info_dict,
|
||||
'_type': 'playlist',
|
||||
'entries': entries,
|
||||
}
|
||||
else:
|
||||
description = self._og_search_description(webpage, default=None)
|
||||
uploader_id = traverse_obj(post_json, 'reblogged_root_name', 'blog_name')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': post_json.get('summary') or (blog if api_only else self._html_search_regex(
|
||||
r'(?s)<title>(?P<title>.*?)(?: \| Tumblr)?</title>', webpage, 'title')),
|
||||
'description': description,
|
||||
'thumbnail': (traverse_obj(video_json, ('poster', 0, 'url'))
|
||||
or self._og_search_thumbnail(webpage, default=None)),
|
||||
'uploader_id': uploader_id,
|
||||
'uploader_url': f'https://{uploader_id}.tumblr.com/' if uploader_id else None,
|
||||
'duration': duration,
|
||||
'like_count': post_json.get('like_count'),
|
||||
'repost_count': post_json.get('reblog_count'),
|
||||
'age_limit': {True: 18, False: 0}.get(post_json.get('is_nsfw')),
|
||||
'tags': post_json.get('tags'),
|
||||
'formats': formats,
|
||||
}
|
||||
return {
|
||||
**info_dict,
|
||||
**entries[0],
|
||||
}
|
||||
|
|
|
@ -147,7 +147,7 @@ class WrestleUniverseBaseIE(InfoExtractor):
|
|||
metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False)
|
||||
if not metadata:
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
nextjs_data = self._search_nextjs_data(webpage, video_id)
|
||||
nextjs_data = self._search_nextjs_data(webpage, video_id, fatal=False)
|
||||
metadata = traverse_obj(nextjs_data, (
|
||||
'props', 'pageProps', *variadic(props_keys, (str, bytes, dict, set)), {dict})) or {}
|
||||
return metadata
|
||||
|
|
Loading…
Reference in New Issue