mirror of https://github.com/yt-dlp/yt-dlp.git
397 lines
16 KiB
Python
397 lines
16 KiB
Python
import base64
|
|
import json
|
|
import re
|
|
import urllib.parse
|
|
|
|
from .adobepass import AdobePassIE
|
|
from .common import InfoExtractor
|
|
from .once import OnceIE
|
|
from ..utils import (
|
|
determine_ext,
|
|
dict_get,
|
|
int_or_none,
|
|
unified_strdate,
|
|
unified_timestamp,
|
|
)
|
|
|
|
|
|
class ESPNIE(OnceIE):
|
|
_VALID_URL = r'''(?x)
|
|
https?://
|
|
(?:
|
|
(?:
|
|
(?:
|
|
(?:(?:\w+\.)+)?espn\.go|
|
|
(?:www\.)?espn
|
|
)\.com/
|
|
(?:
|
|
(?:
|
|
video/(?:clip|iframe/twitter)|
|
|
)
|
|
(?:
|
|
.*?\?.*?\bid=|
|
|
/_/id/
|
|
)|
|
|
[^/]+/video/
|
|
)
|
|
)|
|
|
(?:www\.)espnfc\.(?:com|us)/(?:video/)?[^/]+/\d+/video/
|
|
)
|
|
(?P<id>\d+)
|
|
'''
|
|
|
|
_TESTS = [{
|
|
'url': 'http://espn.go.com/video/clip?id=10365079',
|
|
'info_dict': {
|
|
'id': '10365079',
|
|
'ext': 'mp4',
|
|
'title': '30 for 30 Shorts: Judging Jewell',
|
|
'description': 'md5:39370c2e016cb4ecf498ffe75bef7f0f',
|
|
'timestamp': 1390936111,
|
|
'upload_date': '20140128',
|
|
'duration': 1302,
|
|
'thumbnail': r're:https://.+\.jpg',
|
|
},
|
|
'params': {
|
|
'skip_download': True,
|
|
},
|
|
}, {
|
|
'url': 'https://broadband.espn.go.com/video/clip?id=18910086',
|
|
'info_dict': {
|
|
'id': '18910086',
|
|
'ext': 'mp4',
|
|
'title': 'Kyrie spins around defender for two',
|
|
'description': 'md5:2b0f5bae9616d26fba8808350f0d2b9b',
|
|
'timestamp': 1489539155,
|
|
'upload_date': '20170315',
|
|
},
|
|
'params': {
|
|
'skip_download': True,
|
|
},
|
|
'expected_warnings': ['Unable to download f4m manifest'],
|
|
}, {
|
|
'url': 'http://nonredline.sports.espn.go.com/video/clip?id=19744672',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'https://cdn.espn.go.com/video/clip/_/id/19771774',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'http://www.espn.com/video/clip?id=10365079',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'http://www.espn.com/video/clip/_/id/17989860',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'http://www.espnfc.us/video/espn-fc-tv/86/video/3319154/nashville-unveiled-as-the-newest-club-in-mls',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'http://www.espnfc.com/english-premier-league/23/video/3324163/premier-league-in-90-seconds-golden-tweets',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'http://www.espn.com/espnw/video/26066627/arkansas-gibson-completes-hr-cycle-four-innings',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'http://www.espn.com/watch/player?id=19141491',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'http://www.espn.com/watch/player?bucketId=257&id=19505875',
|
|
'only_matching': True,
|
|
}, ]
|
|
|
|
def _real_extract(self, url):
|
|
video_id = self._match_id(url)
|
|
|
|
clip = self._download_json(
|
|
'http://api-app.espn.com/v1/video/clips/%s' % video_id,
|
|
video_id)['videos'][0]
|
|
|
|
title = clip['headline']
|
|
|
|
format_urls = set()
|
|
formats = []
|
|
|
|
def traverse_source(source, base_source_id=None):
|
|
for source_id, source in source.items():
|
|
if source_id == 'alert':
|
|
continue
|
|
elif isinstance(source, str):
|
|
extract_source(source, base_source_id)
|
|
elif isinstance(source, dict):
|
|
traverse_source(
|
|
source,
|
|
'%s-%s' % (base_source_id, source_id)
|
|
if base_source_id else source_id)
|
|
|
|
def extract_source(source_url, source_id=None):
|
|
if source_url in format_urls:
|
|
return
|
|
format_urls.add(source_url)
|
|
ext = determine_ext(source_url)
|
|
if OnceIE.suitable(source_url):
|
|
formats.extend(self._extract_once_formats(source_url))
|
|
elif ext == 'smil':
|
|
formats.extend(self._extract_smil_formats(
|
|
source_url, video_id, fatal=False))
|
|
elif ext == 'f4m':
|
|
formats.extend(self._extract_f4m_formats(
|
|
source_url, video_id, f4m_id=source_id, fatal=False))
|
|
elif ext == 'm3u8':
|
|
formats.extend(self._extract_m3u8_formats(
|
|
source_url, video_id, 'mp4', entry_protocol='m3u8_native',
|
|
m3u8_id=source_id, fatal=False))
|
|
else:
|
|
f = {
|
|
'url': source_url,
|
|
'format_id': source_id,
|
|
}
|
|
mobj = re.search(r'(\d+)p(\d+)_(\d+)k\.', source_url)
|
|
if mobj:
|
|
f.update({
|
|
'height': int(mobj.group(1)),
|
|
'fps': int(mobj.group(2)),
|
|
'tbr': int(mobj.group(3)),
|
|
})
|
|
if source_id == 'mezzanine':
|
|
f['quality'] = 1
|
|
formats.append(f)
|
|
|
|
links = clip.get('links', {})
|
|
traverse_source(links.get('source', {}))
|
|
traverse_source(links.get('mobile', {}))
|
|
self._sort_formats(formats)
|
|
|
|
description = clip.get('caption') or clip.get('description')
|
|
thumbnail = clip.get('thumbnail')
|
|
duration = int_or_none(clip.get('duration'))
|
|
timestamp = unified_timestamp(clip.get('originalPublishDate'))
|
|
|
|
return {
|
|
'id': video_id,
|
|
'title': title,
|
|
'description': description,
|
|
'thumbnail': thumbnail,
|
|
'timestamp': timestamp,
|
|
'duration': duration,
|
|
'formats': formats,
|
|
}
|
|
|
|
|
|
class ESPNArticleIE(InfoExtractor):
|
|
_VALID_URL = r'https?://(?:espn\.go|(?:www\.)?espn)\.com/(?:[^/]+/)*(?P<id>[^/]+)'
|
|
_TESTS = [{
|
|
'url': 'http://espn.go.com/nba/recap?gameId=400793786',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'http://espn.go.com/blog/golden-state-warriors/post/_/id/593/how-warriors-rapidly-regained-a-winning-edge',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'http://espn.go.com/sports/endurance/story/_/id/12893522/dzhokhar-tsarnaev-sentenced-role-boston-marathon-bombings',
|
|
'only_matching': True,
|
|
}, {
|
|
'url': 'http://espn.go.com/nba/playoffs/2015/story/_/id/12887571/john-wall-washington-wizards-no-swelling-left-hand-wrist-game-5-return',
|
|
'only_matching': True,
|
|
}]
|
|
|
|
@classmethod
|
|
def suitable(cls, url):
|
|
return False if (ESPNIE.suitable(url) or WatchESPNIE.suitable(url)) else super().suitable(url)
|
|
|
|
def _real_extract(self, url):
|
|
video_id = self._match_id(url)
|
|
|
|
webpage = self._download_webpage(url, video_id)
|
|
|
|
video_id = self._search_regex(
|
|
r'class=(["\']).*?video-play-button.*?\1[^>]+data-id=["\'](?P<id>\d+)',
|
|
webpage, 'video id', group='id')
|
|
|
|
return self.url_result(
|
|
'http://espn.go.com/video/clip?id=%s' % video_id, ESPNIE.ie_key())
|
|
|
|
|
|
class FiveThirtyEightIE(InfoExtractor):
|
|
_VALID_URL = r'https?://(?:www\.)?fivethirtyeight\.com/features/(?P<id>[^/?#]+)'
|
|
_TEST = {
|
|
'url': 'http://fivethirtyeight.com/features/how-the-6-8-raiders-can-still-make-the-playoffs/',
|
|
'info_dict': {
|
|
'id': '56032156',
|
|
'ext': 'flv',
|
|
'title': 'FiveThirtyEight: The Raiders can still make the playoffs',
|
|
'description': 'Neil Paine breaks down the simplest scenario that will put the Raiders into the playoffs at 8-8.',
|
|
},
|
|
'params': {
|
|
'skip_download': True,
|
|
},
|
|
}
|
|
|
|
def _real_extract(self, url):
|
|
video_id = self._match_id(url)
|
|
|
|
webpage = self._download_webpage(url, video_id)
|
|
|
|
embed_url = self._search_regex(
|
|
r'<iframe[^>]+src=["\'](https?://fivethirtyeight\.abcnews\.go\.com/video/embed/\d+/\d+)',
|
|
webpage, 'embed url')
|
|
|
|
return self.url_result(embed_url, 'AbcNewsVideo')
|
|
|
|
|
|
class ESPNCricInfoIE(InfoExtractor):
|
|
_VALID_URL = r'https?://(?:www\.)?espncricinfo\.com/video/[^#$&?/]+-(?P<id>\d+)'
|
|
_TESTS = [{
|
|
'url': 'https://www.espncricinfo.com/video/finch-chasing-comes-with-risks-despite-world-cup-trend-1289135',
|
|
'info_dict': {
|
|
'id': '1289135',
|
|
'ext': 'mp4',
|
|
'title': 'Finch: Chasing comes with \'risks\' despite World Cup trend',
|
|
'description': 'md5:ea32373303e25efbb146efdfc8a37829',
|
|
'upload_date': '20211113',
|
|
'duration': 96,
|
|
},
|
|
'params': {'skip_download': True}
|
|
}]
|
|
|
|
def _real_extract(self, url):
|
|
id = self._match_id(url)
|
|
data_json = self._download_json(f'https://hs-consumer-api.espncricinfo.com/v1/pages/video/video-details?videoId={id}', id)['video']
|
|
formats, subtitles = [], {}
|
|
for item in data_json.get('playbacks') or []:
|
|
if item.get('type') == 'HLS' and item.get('url'):
|
|
m3u8_frmts, m3u8_subs = self._extract_m3u8_formats_and_subtitles(item['url'], id)
|
|
formats.extend(m3u8_frmts)
|
|
subtitles = self._merge_subtitles(subtitles, m3u8_subs)
|
|
elif item.get('type') == 'AUDIO' and item.get('url'):
|
|
formats.append({
|
|
'url': item['url'],
|
|
'vcodec': 'none',
|
|
})
|
|
self._sort_formats(formats)
|
|
return {
|
|
'id': id,
|
|
'title': data_json.get('title'),
|
|
'description': data_json.get('summary'),
|
|
'upload_date': unified_strdate(dict_get(data_json, ('publishedAt', 'recordedAt'))),
|
|
'duration': data_json.get('duration'),
|
|
'formats': formats,
|
|
'subtitles': subtitles,
|
|
}
|
|
|
|
|
|
class WatchESPNIE(AdobePassIE):
|
|
_VALID_URL = r'https://www.espn.com/watch/player/_/id/(?P<id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})'
|
|
_TESTS = [{
|
|
'url': 'https://www.espn.com/watch/player/_/id/ba7d17da-453b-4697-bf92-76a99f61642b',
|
|
'info_dict': {
|
|
'id': 'ba7d17da-453b-4697-bf92-76a99f61642b',
|
|
'ext': 'mp4',
|
|
'title': 'Serbia vs. Turkey',
|
|
'thumbnail': 'https://artwork.api.espn.com/artwork/collections/media/ba7d17da-453b-4697-bf92-76a99f61642b/default?width=640&apikey=1ngjw23osgcis1i1vbj96lmfqs',
|
|
},
|
|
'params': {
|
|
'skip_download': True,
|
|
},
|
|
}, {
|
|
'url': 'https://www.espn.com/watch/player/_/id/4e9b5bd1-4ceb-4482-9d28-1dd5f30d2f34',
|
|
'info_dict': {
|
|
'id': '4e9b5bd1-4ceb-4482-9d28-1dd5f30d2f34',
|
|
'ext': 'mp4',
|
|
'title': 'Real Madrid vs. Real Betis (LaLiga)',
|
|
'thumbnail': 'https://s.secure.espncdn.com/stitcher/artwork/collections/media/bd1f3d12-0654-47d9-852e-71b85ea695c7/16x9.jpg?timestamp=202201112217&showBadge=true&cb=12&package=ESPN_PLUS',
|
|
},
|
|
'params': {
|
|
'skip_download': True,
|
|
},
|
|
}]
|
|
|
|
_API_KEY = 'ZXNwbiZicm93c2VyJjEuMC4w.ptUt7QxsteaRruuPmGZFaJByOoqKvDP2a5YkInHrc7c'
|
|
|
|
def _call_bamgrid_api(self, path, video_id, payload=None, headers={}):
|
|
if 'Authorization' not in headers:
|
|
headers['Authorization'] = f'Bearer {self._API_KEY}'
|
|
parse = urllib.parse.urlencode if path == 'token' else json.dumps
|
|
return self._download_json(
|
|
f'https://espn.api.edge.bamgrid.com/{path}', video_id, headers=headers, data=parse(payload).encode())
|
|
|
|
def _real_extract(self, url):
|
|
video_id = self._match_id(url)
|
|
video_data = self._download_json(
|
|
f'https://watch-cdn.product.api.espn.com/api/product/v3/watchespn/web/playback/event?id={video_id}',
|
|
video_id)['playbackState']
|
|
|
|
# ESPN+ subscription required, through cookies
|
|
if 'DTC' in video_data.get('sourceId'):
|
|
cookie = self._get_cookies(url).get('ESPN-ONESITE.WEB-PROD.token')
|
|
if not cookie:
|
|
self.raise_login_required(method='cookies')
|
|
|
|
assertion = self._call_bamgrid_api(
|
|
'devices', video_id,
|
|
headers={'Content-Type': 'application/json; charset=UTF-8'},
|
|
payload={
|
|
'deviceFamily': 'android',
|
|
'applicationRuntime': 'android',
|
|
'deviceProfile': 'tv',
|
|
'attributes': {},
|
|
})['assertion']
|
|
token = self._call_bamgrid_api(
|
|
'token', video_id, payload={
|
|
'subject_token': assertion,
|
|
'subject_token_type': 'urn:bamtech:params:oauth:token-type:device',
|
|
'platform': 'android',
|
|
'grant_type': 'urn:ietf:params:oauth:grant-type:token-exchange'
|
|
})['access_token']
|
|
|
|
assertion = self._call_bamgrid_api(
|
|
'accounts/grant', video_id, payload={'id_token': cookie.value.split('|')[1]},
|
|
headers={
|
|
'Authorization': token,
|
|
'Content-Type': 'application/json; charset=UTF-8'
|
|
})['assertion']
|
|
token = self._call_bamgrid_api(
|
|
'token', video_id, payload={
|
|
'subject_token': assertion,
|
|
'subject_token_type': 'urn:bamtech:params:oauth:token-type:account',
|
|
'platform': 'android',
|
|
'grant_type': 'urn:ietf:params:oauth:grant-type:token-exchange'
|
|
})['access_token']
|
|
|
|
playback = self._download_json(
|
|
video_data['videoHref'].format(scenario='browser~ssai'), video_id,
|
|
headers={
|
|
'Accept': 'application/vnd.media-service+json; version=5',
|
|
'Authorization': token
|
|
})
|
|
m3u8_url, headers = playback['stream']['complete'][0]['url'], {'authorization': token}
|
|
|
|
# No login required
|
|
elif video_data.get('sourceId') == 'ESPN_FREE':
|
|
asset = self._download_json(
|
|
f'https://watch.auth.api.espn.com/video/auth/media/{video_id}/asset?apikey=uiqlbgzdwuru14v627vdusswb',
|
|
video_id)
|
|
m3u8_url, headers = asset['stream'], {}
|
|
|
|
# TV Provider required
|
|
else:
|
|
resource = self._get_mvpd_resource('ESPN', video_data['name'], video_id, None)
|
|
auth = self._extract_mvpd_auth(url, video_id, 'ESPN', resource).encode()
|
|
|
|
asset = self._download_json(
|
|
f'https://watch.auth.api.espn.com/video/auth/media/{video_id}/asset?apikey=uiqlbgzdwuru14v627vdusswb',
|
|
video_id, data=f'adobeToken={urllib.parse.quote_plus(base64.b64encode(auth))}&drmSupport=HLS'.encode())
|
|
m3u8_url, headers = asset['stream'], {}
|
|
|
|
formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
|
|
self._sort_formats(formats)
|
|
|
|
return {
|
|
'id': video_id,
|
|
'title': video_data.get('name'),
|
|
'formats': formats,
|
|
'subtitles': subtitles,
|
|
'thumbnail': video_data.get('posterHref'),
|
|
'http_headers': headers,
|
|
}
|