mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-11-17 01:25:11 +00:00
[extractor/instagram] Fix extraction (#4696)
Closes #4657, #4532, #4475 Authored by: bashonly, pritam20ps05
This commit is contained in:
bashonly
GitHub
parent
2b3e43e247
commit
7d3b98be4c
1 changed files with 97 additions and 71 deletions
|
|
@ -39,37 +39,42 @@ class InstagramBaseIE(InfoExtractor):
|
||||||
_NETRC_MACHINE = 'instagram'
|
_NETRC_MACHINE = 'instagram'
|
||||||
_IS_LOGGED_IN = False
|
_IS_LOGGED_IN = False
|
||||||
|
|
||||||
|
_API_BASE_URL = 'https://i.instagram.com/api/v1'
|
||||||
|
_LOGIN_URL = 'https://www.instagram.com/accounts/login'
|
||||||
|
_API_HEADERS = {
|
||||||
|
'X-IG-App-ID': '936619743392459',
|
||||||
|
'X-ASBD-ID': '198387',
|
||||||
|
'X-IG-WWW-Claim': '0',
|
||||||
|
'Origin': 'https://www.instagram.com',
|
||||||
|
'Accept': '*/*',
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36',
|
||||||
|
}
|
||||||
|
|
||||||
def _perform_login(self, username, password):
|
def _perform_login(self, username, password):
|
||||||
if self._IS_LOGGED_IN:
|
if self._IS_LOGGED_IN:
|
||||||
return
|
return
|
||||||
|
|
||||||
login_webpage = self._download_webpage(
|
login_webpage = self._download_webpage(
|
||||||
'https://www.instagram.com/accounts/login/', None,
|
self._LOGIN_URL, None, note='Downloading login webpage', errnote='Failed to download login webpage')
|
||||||
note='Downloading login webpage', errnote='Failed to download login webpage')
|
|
||||||
|
|
||||||
shared_data = self._parse_json(
|
shared_data = self._parse_json(self._search_regex(
|
||||||
self._search_regex(
|
r'window\._sharedData\s*=\s*({.+?});', login_webpage, 'shared data', default='{}'), None)
|
||||||
r'window\._sharedData\s*=\s*({.+?});',
|
|
||||||
login_webpage, 'shared data', default='{}'),
|
|
||||||
None)
|
|
||||||
|
|
||||||
login = self._download_json('https://www.instagram.com/accounts/login/ajax/', None, note='Logging in', headers={
|
login = self._download_json(
|
||||||
'Accept': '*/*',
|
f'{self._LOGIN_URL}/ajax/', None, note='Logging in', headers={
|
||||||
'X-IG-App-ID': '936619743392459',
|
**self._API_HEADERS,
|
||||||
'X-ASBD-ID': '198387',
|
'X-Requested-With': 'XMLHttpRequest',
|
||||||
'X-IG-WWW-Claim': '0',
|
'X-CSRFToken': shared_data['config']['csrf_token'],
|
||||||
'X-Requested-With': 'XMLHttpRequest',
|
'X-Instagram-AJAX': shared_data['rollout_hash'],
|
||||||
'X-CSRFToken': shared_data['config']['csrf_token'],
|
'Referer': 'https://www.instagram.com/',
|
||||||
'X-Instagram-AJAX': shared_data['rollout_hash'],
|
}, data=urlencode_postdata({
|
||||||
'Referer': 'https://www.instagram.com/',
|
'enc_password': f'#PWD_INSTAGRAM_BROWSER:0:{int(time.time())}:{password}',
|
||||||
}, data=urlencode_postdata({
|
'username': username,
|
||||||
'enc_password': f'#PWD_INSTAGRAM_BROWSER:0:{int(time.time())}:{password}',
|
'queryParams': '{}',
|
||||||
'username': username,
|
'optIntoOneTap': 'false',
|
||||||
'queryParams': '{}',
|
'stopDeletionNonce': '',
|
||||||
'optIntoOneTap': 'false',
|
'trustedDeviceRecords': '{}',
|
||||||
'stopDeletionNonce': '',
|
}))
|
||||||
'trustedDeviceRecords': '{}',
|
|
||||||
}))
|
|
||||||
|
|
||||||
if not login.get('authenticated'):
|
if not login.get('authenticated'):
|
||||||
if login.get('message'):
|
if login.get('message'):
|
||||||
|
|
@ -134,7 +139,7 @@ def _extract_nodes(self, nodes, is_direct=False):
|
||||||
}
|
}
|
||||||
|
|
||||||
def _extract_product_media(self, product_media):
|
def _extract_product_media(self, product_media):
|
||||||
media_id = product_media.get('code') or product_media.get('id')
|
media_id = product_media.get('code') or _pk_to_id(product_media.get('pk'))
|
||||||
vcodec = product_media.get('video_codec')
|
vcodec = product_media.get('video_codec')
|
||||||
dash_manifest_raw = product_media.get('video_dash_manifest')
|
dash_manifest_raw = product_media.get('video_dash_manifest')
|
||||||
videos_list = product_media.get('video_versions')
|
videos_list = product_media.get('video_versions')
|
||||||
|
|
@ -179,7 +184,7 @@ def _extract_product(self, product_info):
|
||||||
|
|
||||||
user_info = product_info.get('user') or {}
|
user_info = product_info.get('user') or {}
|
||||||
info_dict = {
|
info_dict = {
|
||||||
'id': product_info.get('code') or product_info.get('id'),
|
'id': product_info.get('code') or _pk_to_id(product_info.get('pk')),
|
||||||
'title': product_info.get('title') or f'Video by {user_info.get("username")}',
|
'title': product_info.get('title') or f'Video by {user_info.get("username")}',
|
||||||
'description': traverse_obj(product_info, ('caption', 'text'), expected_type=str_or_none),
|
'description': traverse_obj(product_info, ('caption', 'text'), expected_type=str_or_none),
|
||||||
'timestamp': int_or_none(product_info.get('taken_at')),
|
'timestamp': int_or_none(product_info.get('taken_at')),
|
||||||
|
|
@ -360,49 +365,74 @@ def _extract_embed_urls(cls, url, webpage):
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id, url = self._match_valid_url(url).group('id', 'url')
|
video_id, url = self._match_valid_url(url).group('id', 'url')
|
||||||
general_info = self._download_json(
|
media, webpage = {}, ''
|
||||||
f'https://www.instagram.com/graphql/query/?query_hash=9f8827793ef34641b2fb195d4d41151c'
|
|
||||||
f'&variables=%7B"shortcode":"{video_id}",'
|
api_check = self._download_json(
|
||||||
'"parent_comment_count":10,"has_threaded_comments":true}', video_id, fatal=False, errnote=False,
|
f'{self._API_BASE_URL}/web/get_ruling_for_content/?content_type=MEDIA&target_id={_id_to_pk(video_id)}',
|
||||||
headers={
|
video_id, headers=self._API_HEADERS, fatal=False, note='Setting up session', errnote=False) or {}
|
||||||
'Accept': '*',
|
csrf_token = self._get_cookies('https://www.instagram.com').get('csrftoken')
|
||||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
|
|
||||||
'Authority': 'www.instagram.com',
|
if not csrf_token:
|
||||||
'Referer': 'https://www.instagram.com',
|
self.report_warning('No csrf token set by Instagram API', video_id)
|
||||||
'x-ig-app-id': '936619743392459',
|
elif api_check.get('status') != 'ok':
|
||||||
})
|
self.report_warning('Instagram API is not granting access', video_id)
|
||||||
media = traverse_obj(general_info, ('data', 'shortcode_media')) or {}
|
else:
|
||||||
|
if self._get_cookies(url).get('sessionid'):
|
||||||
|
media = traverse_obj(self._download_json(
|
||||||
|
f'{self._API_BASE_URL}/media/{_id_to_pk(video_id)}/info/', video_id,
|
||||||
|
fatal=False, note='Downloading video info', headers={
|
||||||
|
**self._API_HEADERS,
|
||||||
|
'X-CSRFToken': csrf_token.value,
|
||||||
|
}), ('items', 0))
|
||||||
|
if media:
|
||||||
|
return self._extract_product(media)
|
||||||
|
|
||||||
|
variables = {
|
||||||
|
'shortcode': video_id,
|
||||||
|
'child_comment_count': 3,
|
||||||
|
'fetch_comment_count': 40,
|
||||||
|
'parent_comment_count': 24,
|
||||||
|
'has_threaded_comments': True,
|
||||||
|
}
|
||||||
|
general_info = self._download_json(
|
||||||
|
'https://www.instagram.com/graphql/query/', video_id, fatal=False,
|
||||||
|
headers={
|
||||||
|
**self._API_HEADERS,
|
||||||
|
'X-CSRFToken': csrf_token.value,
|
||||||
|
'X-Requested-With': 'XMLHttpRequest',
|
||||||
|
'Referer': url,
|
||||||
|
}, query={
|
||||||
|
'query_hash': '9f8827793ef34641b2fb195d4d41151c',
|
||||||
|
'variables': json.dumps(variables, separators=(',', ':')),
|
||||||
|
})
|
||||||
|
media = traverse_obj(general_info, ('data', 'shortcode_media'))
|
||||||
|
|
||||||
if not media:
|
if not media:
|
||||||
self.report_warning('General metadata extraction failed', video_id)
|
self.report_warning('General metadata extraction failed (some metadata might be missing).', video_id)
|
||||||
|
webpage, urlh = self._download_webpage_handle(url, video_id)
|
||||||
|
shared_data = self._search_json(
|
||||||
|
r'window\._sharedData\s*=', webpage, 'shared data', video_id, fatal=False)
|
||||||
|
|
||||||
info = self._download_json(
|
if self._LOGIN_URL not in urlh.geturl():
|
||||||
f'https://i.instagram.com/api/v1/media/{_id_to_pk(video_id)}/info/', video_id,
|
media.update(traverse_obj(
|
||||||
fatal=False, note='Downloading video info', errnote=False, headers={
|
shared_data, ('entry_data', 'PostPage', 0, 'graphql', 'shortcode_media'),
|
||||||
'Accept': '*',
|
('entry_data', 'PostPage', 0, 'media'), expected_type=dict) or {})
|
||||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
|
else:
|
||||||
'Authority': 'www.instagram.com',
|
self.report_warning('Main webpage is locked behind the login page. Retrying with embed webpage')
|
||||||
'Referer': 'https://www.instagram.com',
|
webpage = self._download_webpage(
|
||||||
'x-ig-app-id': '936619743392459',
|
f'{url}/embed/', video_id, note='Downloading embed webpage', fatal=False)
|
||||||
})
|
additional_data = self._search_json(
|
||||||
if info:
|
r'window\.__additionalDataLoaded\s*\(\s*[^,]+,\s*', webpage, 'additional data', video_id, fatal=False)
|
||||||
media.update(info['items'][0])
|
if not additional_data:
|
||||||
return self._extract_product(media)
|
self.raise_login_required('Requested content was not found, the content might be private')
|
||||||
|
|
||||||
webpage = self._download_webpage(
|
product_item = traverse_obj(additional_data, ('items', 0), expected_type=dict)
|
||||||
f'https://www.instagram.com/p/{video_id}/embed/', video_id,
|
if product_item:
|
||||||
note='Downloading embed webpage', fatal=False)
|
media.update(product_item)
|
||||||
if not webpage:
|
return self._extract_product(media)
|
||||||
self.raise_login_required('Requested content was not found, the content might be private')
|
|
||||||
|
|
||||||
additional_data = self._search_json(
|
media.update(traverse_obj(
|
||||||
r'window\.__additionalDataLoaded\s*\(\s*[^,]+,\s*', webpage, 'additional data', video_id, fatal=False)
|
additional_data, ('graphql', 'shortcode_media'), 'shortcode_media', expected_type=dict) or {})
|
||||||
product_item = traverse_obj(additional_data, ('items', 0), expected_type=dict)
|
|
||||||
if product_item:
|
|
||||||
media.update(product_item)
|
|
||||||
return self._extract_product(media)
|
|
||||||
|
|
||||||
media.update(traverse_obj(
|
|
||||||
additional_data, ('graphql', 'shortcode_media'), 'shortcode_media', expected_type=dict) or {})
|
|
||||||
|
|
||||||
username = traverse_obj(media, ('owner', 'username')) or self._search_regex(
|
username = traverse_obj(media, ('owner', 'username')) or self._search_regex(
|
||||||
r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"', webpage, 'username', fatal=False)
|
r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"', webpage, 'username', fatal=False)
|
||||||
|
|
@ -649,12 +679,8 @@ def _real_extract(self, url):
|
||||||
|
|
||||||
story_info_url = user_id if username != 'highlights' else f'highlight:{story_id}'
|
story_info_url = user_id if username != 'highlights' else f'highlight:{story_id}'
|
||||||
videos = traverse_obj(self._download_json(
|
videos = traverse_obj(self._download_json(
|
||||||
f'https://i.instagram.com/api/v1/feed/reels_media/?reel_ids={story_info_url}',
|
f'{self._API_BASE_URL}/feed/reels_media/?reel_ids={story_info_url}',
|
||||||
story_id, errnote=False, fatal=False, headers={
|
story_id, errnote=False, fatal=False, headers=self._API_HEADERS), 'reels')
|
||||||
'X-IG-App-ID': 936619743392459,
|
|
||||||
'X-ASBD-ID': 198387,
|
|
||||||
'X-IG-WWW-Claim': 0,
|
|
||||||
}), 'reels')
|
|
||||||
if not videos:
|
if not videos:
|
||||||
self.raise_login_required('You need to log in to access this content')
|
self.raise_login_required('You need to log in to access this content')
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue