etc/yt-dlp
0
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-30 03:33:02 +00:00
Code Issues Projects Releases Wiki Activity

[youtube] simplify and de-duplicate client definitions (#577)

Browse source
This commit is contained in:
pukkandan 2021-07-31 01:15:04 +05:30
parent 9275f62cf8
commit 000c15a4ca
No known key found for this signature in database
GPG key ID: 0F00D95A001F4698
1 changed files with 157 additions and 249 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

406
yt_dlp/extractor/youtube.py
View file

@ -67,6 +67,137 @@ def parse_qs(url):
return compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query) return compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
# any clients starting with _ cannot be explicity requested by the user
INNERTUBE_CLIENTS = {
'web': {
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB',
'clientVersion': '2.20210622.10.00',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 1
},
'web_embedded': {
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB_EMBEDDED_PLAYER',
'clientVersion': '1.20210620.0.1',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 56
},
'web_music': {
'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
'INNERTUBE_HOST': 'music.youtube.com',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB_REMIX',
'clientVersion': '1.20210621.00.00',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
},
'android': {
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'ANDROID',
'clientVersion': '16.20',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
},
'android_embedded': {
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'ANDROID_EMBEDDED_PLAYER',
'clientVersion': '16.20',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 55
},
'android_music': {
'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
'INNERTUBE_HOST': 'music.youtube.com',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'ANDROID_MUSIC',
'clientVersion': '4.32',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
},
'ios': {
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'IOS',
'clientVersion': '16.20',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 5
},
'ios_embedded': {
'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'IOS_MESSAGES_EXTENSION',
'clientVersion': '16.20',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 66
},
'ios_music': {
'INNERTUBE_API_KEY': 'AIzaSyDK3iBpDP9nHVTk2qL73FLJICfOC3c51Og',
'INNERTUBE_HOST': 'music.youtube.com',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'IOS_MUSIC',
'clientVersion': '4.32',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 26
},
'mweb': {
'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'MWEB',
'clientVersion': '2.20210721.07.00',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 2
},
}
def build_innertube_clients():
base_clients = ('android', 'web', 'ios', 'mweb')
priority = qualities(base_clients[::-1])
for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM4DrUqRUYnGn3llEO78bcxq8')
ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
ytcfg['priority'] = 10 * priority(client.split('_', 1)[0])
if client in base_clients:
INNERTUBE_CLIENTS[f'{client}_agegate'] = agegate_ytcfg = copy.deepcopy(ytcfg)
agegate_ytcfg['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
agegate_ytcfg['priority'] -= 1
elif client.endswith('_embedded'):
ytcfg['priority'] -= 2
else:
ytcfg['priority'] -= 3
build_innertube_clients()
class YoutubeBaseInfoExtractor(InfoExtractor): class YoutubeBaseInfoExtractor(InfoExtractor):
"""Provide base functions for Youtube extractors""" """Provide base functions for Youtube extractors"""
_LOGIN_URL = 'https://accounts.google.com/ServiceLogin' _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
@ -312,250 +443,22 @@ def _real_initialize(self):
_YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;' _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
_YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)' _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
_YT_DEFAULT_YTCFGS = { def _get_default_ytcfg(self, client='web'):
'WEB': { return copy.deepcopy(INNERTUBE_CLIENTS[client])
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'WEB',
'INNERTUBE_CLIENT_VERSION': '2.20210622.10.00',
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB',
'clientVersion': '2.20210622.10.00',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 1
},
'WEB_AGEGATE': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'WEB',
'INNERTUBE_CLIENT_VERSION': '2.20210622.10.00',
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB',
'clientVersion': '2.20210622.10.00',
'clientScreen': 'EMBED',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 1
},
'WEB_REMIX': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'WEB_REMIX',
'INNERTUBE_CLIENT_VERSION': '1.20210621.00.00',
'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB_REMIX',
'clientVersion': '1.20210621.00.00',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 67
},
'WEB_EMBEDDED_PLAYER': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'WEB_EMBEDDED_PLAYER',
'INNERTUBE_CLIENT_VERSION': '1.20210620.0.1',
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB_EMBEDDED_PLAYER',
'clientVersion': '1.20210620.0.1',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 56
},
'ANDROID': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'ANDROID',
'INNERTUBE_CLIENT_VERSION': '16.20',
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'ANDROID',
'clientVersion': '16.20',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 3
},
'ANDROID_AGEGATE': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'ANDROID',
'INNERTUBE_CLIENT_VERSION': '16.20',
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'ANDROID',
'clientVersion': '16.20',
'clientScreen': 'EMBED',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 3
},
'ANDROID_EMBEDDED_PLAYER': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'ANDROID_EMBEDDED_PLAYER',
'INNERTUBE_CLIENT_VERSION': '16.20',
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'ANDROID_EMBEDDED_PLAYER',
'clientVersion': '16.20',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 55
},
'ANDROID_MUSIC': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'ANDROID_MUSIC',
'INNERTUBE_CLIENT_VERSION': '4.32',
'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'ANDROID_MUSIC',
'clientVersion': '4.32',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 21
},
'IOS': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'IOS',
'INNERTUBE_CLIENT_VERSION': '16.20',
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'IOS',
'clientVersion': '16.20',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 5
},
'IOS_AGEGATE': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'IOS',
'INNERTUBE_CLIENT_VERSION': '16.20',
'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'IOS',
'clientVersion': '16.20',
'clientScreen': 'EMBED',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 5
},
'IOS_MUSIC': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'IOS_MUSIC',
'INNERTUBE_CLIENT_VERSION': '4.32',
'INNERTUBE_API_KEY': 'AIzaSyDK3iBpDP9nHVTk2qL73FLJICfOC3c51Og',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'IOS_MUSIC',
'clientVersion': '4.32',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 26
},
'IOS_MESSAGES_EXTENSION': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'IOS_MESSAGES_EXTENSION',
'INNERTUBE_CLIENT_VERSION': '16.20',
'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'IOS_MESSAGES_EXTENSION',
'clientVersion': '16.20',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 66
},
'MWEB': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'MWEB',
'INNERTUBE_CLIENT_VERSION': '2.20210721.07.00',
'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'MWEB',
'clientVersion': '2.20210721.07.00',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 2
},
'MWEB_AGEGATE': {
'INNERTUBE_API_VERSION': 'v1',
'INNERTUBE_CLIENT_NAME': 'MWEB',
'INNERTUBE_CLIENT_VERSION': '2.20210721.07.00',
'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'MWEB',
'clientVersion': '2.20210721.07.00',
'clientScreen': 'EMBED',
'hl': 'en',
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 2
},
}
_YT_DEFAULT_INNERTUBE_HOSTS = { def _get_innertube_host(self, client='web'):
'DIRECT': 'youtubei.googleapis.com', return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
'WEB': 'www.youtube.com',
'WEB_REMIX': 'music.youtube.com',
'ANDROID_MUSIC': 'music.youtube.com'
}
# clients starting with _ cannot be explicity requested by the user def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
_YT_CLIENTS = {
'android': 'ANDROID',
'android_music': 'ANDROID_MUSIC',
'android_embedded': 'ANDROID_EMBEDDED_PLAYER',
'android_agegate': 'ANDROID_AGEGATE',
'ios': 'IOS',
'ios_music': 'IOS_MUSIC',
'ios_embedded': 'IOS_MESSAGES_EXTENSION',
'ios_agegate': 'IOS_AGEGATE',
'web': 'WEB',
'web_music': 'WEB_REMIX',
'web_embedded': 'WEB_EMBEDDED_PLAYER',
'web_agegate': 'WEB_AGEGATE',
'mweb': 'MWEB',
'mweb_agegate': 'MWEB_AGEGATE',
}
def _get_default_ytcfg(self, client='WEB'):
if client in self._YT_DEFAULT_YTCFGS:
return copy.deepcopy(self._YT_DEFAULT_YTCFGS[client])
self.write_debug(f'INNERTUBE default client {client} does not exist - falling back to WEB client.')
return copy.deepcopy(self._YT_DEFAULT_YTCFGS['WEB'])
def _get_innertube_host(self, client='WEB'):
return dict_get(self._YT_DEFAULT_INNERTUBE_HOSTS, (client, 'WEB'))
def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='WEB'):
# try_get but with fallback to default ytcfg client values when present # try_get but with fallback to default ytcfg client values when present
_func = lambda y: try_get(y, getter, expected_type) _func = lambda y: try_get(y, getter, expected_type)
return _func(ytcfg) or _func(self._get_default_ytcfg(default_client)) return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
def _extract_client_name(self, ytcfg, default_client='WEB'): def _extract_client_name(self, ytcfg, default_client='web'):
return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CLIENT_NAME'], compat_str, default_client) return (
try_get(ytcfg, lambda x: x['INNERTUBE_CLIENT_NAME'], compat_str)
or self._ytcfg_get_safe(
ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['clientName'], compat_str, default_client))
@staticmethod @staticmethod
def _extract_session_index(*data): def _extract_session_index(*data):
@ -564,13 +467,16 @@ def _extract_session_index(*data):
if session_index is not None: if session_index is not None:
return session_index return session_index
def _extract_client_version(self, ytcfg, default_client='WEB'): def _extract_client_version(self, ytcfg, default_client='web'):
return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CLIENT_VERSION'], compat_str, default_client) return (
try_get(ytcfg, lambda x: x['INNERTUBE_CLIENT_VERSION'], compat_str)
or self._ytcfg_get_safe(
ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion'], compat_str, default_client))
def _extract_api_key(self, ytcfg=None, default_client='WEB'): def _extract_api_key(self, ytcfg=None, default_client='web'):
return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client) return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
def _extract_context(self, ytcfg=None, default_client='WEB'): def _extract_context(self, ytcfg=None, default_client='web'):
_get_context = lambda y: try_get(y, lambda x: x['INNERTUBE_CONTEXT'], dict) _get_context = lambda y: try_get(y, lambda x: x['INNERTUBE_CONTEXT'], dict)
context = _get_context(ytcfg) context = _get_context(ytcfg)
if context: if context:
@ -612,7 +518,7 @@ def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
def _call_api(self, ep, query, video_id, fatal=True, headers=None, def _call_api(self, ep, query, video_id, fatal=True, headers=None,
note='Downloading API JSON', errnote='Unable to download API page', note='Downloading API JSON', errnote='Unable to download API page',
context=None, api_key=None, api_hostname=None, default_client='WEB'): context=None, api_key=None, api_hostname=None, default_client='web'):
data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)} data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
data.update(query) data.update(query)
@ -674,7 +580,7 @@ def extract_ytcfg(self, video_id, webpage):
def generate_api_headers( def generate_api_headers(
self, ytcfg=None, identity_token=None, account_syncid=None, self, ytcfg=None, identity_token=None, account_syncid=None,
visitor_data=None, api_hostname=None, default_client='WEB', session_index=None): visitor_data=None, api_hostname=None, default_client='web', session_index=None):
origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client)) origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
headers = { headers = {
'X-YouTube-Client-Name': compat_str( 'X-YouTube-Client-Name': compat_str(
@ -819,7 +725,7 @@ def _get_text(data, *path_list, max_runs=None):
def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None, def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None, ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
default_client='WEB'): default_client='web'):
response = None response = None
last_error = None last_error = None
count = -1 count = -1
@ -2452,20 +2358,22 @@ def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg,
sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False)
headers = self.generate_api_headers( headers = self.generate_api_headers(
player_ytcfg, identity_token, syncid, player_ytcfg, identity_token, syncid,
default_client=self._YT_CLIENTS[client], session_index=session_index) default_client=client, session_index=session_index)
yt_query = {'videoId': video_id} yt_query = {'videoId': video_id}
yt_query.update(self._generate_player_context(sts)) yt_query.update(self._generate_player_context(sts))
return self._extract_response( return self._extract_response(
item_id=video_id, ep='player', query=yt_query, item_id=video_id, ep='player', query=yt_query,
ytcfg=player_ytcfg, headers=headers, fatal=False, ytcfg=player_ytcfg, headers=headers, fatal=False,
default_client=self._YT_CLIENTS[client], default_client=client,
note='Downloading %s player API JSON' % client.replace('_', ' ').strip() note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
) or None ) or None
def _get_requested_clients(self, url, smuggled_data): def _get_requested_clients(self, url, smuggled_data):
requested_clients = [] requested_clients = []
allowed_clients = [client for client in self._YT_CLIENTS.keys() if client[:1] != '_'] allowed_clients = sorted(
[client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'],
key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
for client in self._configuration_arg('player_client'): for client in self._configuration_arg('player_client'):
if client in allowed_clients: if client in allowed_clients:
requested_clients.append(client) requested_clients.append(client)
@ -2516,7 +2424,7 @@ def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, pl
if self._is_agegated(pr): if self._is_agegated(pr):
client = f'{client}_agegate' client = f'{client}_agegate'
if client in self._YT_CLIENTS and client not in original_clients: if client in INNERTUBE_CLIENTS and client not in original_clients:
clients.append(client) clients.append(client)
# Android player_response does not have microFormats which are needed for # Android player_response does not have microFormats which are needed for