0
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-16 01:25:06 +00:00

Update to ytdl-commit-a08f2b7 (#10012)

[ie] Rework JWPlayer extraction
- f66372403f
[ie/gbnews] Add extractor
- 70f230f9cf
[ie/caffeinetv] Add extractor
- 40bd5c1815
[ie/youporn] Improve extraction
- 0b2ce3685e
[ie/youporn] Add playlist extractors
- 668332b973

Closes #9188, Closes #9523
Authored by: Grub4K, bashonly
This commit is contained in:
Simon Sawicki 2024-05-26 23:09:53 +02:00 committed by GitHub
parent e897bd8292
commit a4da9db87b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 588 additions and 45 deletions

View file

@ -2123,7 +2123,7 @@ # CHANGES FROM YOUTUBE-DL
### New features ### New features
* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@be008e6**](https://github.com/ytdl-org/youtube-dl/commit/be008e657d79832642e2158557c899249c9e31cd) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21)) * Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@a08f2b7**](https://github.com/ytdl-org/youtube-dl/commit/a08f2b7e4567cdc50c0614ee0a4ffdff49b8b6e6) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
* **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API

View file

@ -308,6 +308,7 @@
from .buzzfeed import BuzzFeedIE from .buzzfeed import BuzzFeedIE
from .byutv import BYUtvIE from .byutv import BYUtvIE
from .c56 import C56IE from .c56 import C56IE
from .caffeinetv import CaffeineTVIE
from .callin import CallinIE from .callin import CallinIE
from .caltrans import CaltransIE from .caltrans import CaltransIE
from .cam4 import CAM4IE from .cam4 import CAM4IE
@ -720,6 +721,7 @@
from .gamestar import GameStarIE from .gamestar import GameStarIE
from .gaskrank import GaskrankIE from .gaskrank import GaskrankIE
from .gazeta import GazetaIE from .gazeta import GazetaIE
from .gbnews import GBNewsIE
from .gdcvault import GDCVaultIE from .gdcvault import GDCVaultIE
from .gedidigital import GediDigitalIE from .gedidigital import GediDigitalIE
from .generic import GenericIE from .generic import GenericIE
@ -2501,7 +2503,15 @@
YouNowLiveIE, YouNowLiveIE,
YouNowMomentIE, YouNowMomentIE,
) )
from .youporn import YouPornIE from .youporn import (
YouPornCategoryIE,
YouPornChannelIE,
YouPornCollectionIE,
YouPornIE,
YouPornStarIE,
YouPornTagIE,
YouPornVideosIE,
)
from .zaiko import ( from .zaiko import (
ZaikoETicketIE, ZaikoETicketIE,
ZaikoIE, ZaikoIE,

View file

@ -0,0 +1,74 @@
from .common import InfoExtractor
from ..utils import (
determine_ext,
int_or_none,
parse_iso8601,
traverse_obj,
urljoin,
)
class CaffeineTVIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?caffeine\.tv/[^/?#]+/video/(?P<id>[\da-f-]+)'
_TESTS = [{
'url': 'https://www.caffeine.tv/TsuSurf/video/cffc0a00-e73f-11ec-8080-80017d29f26e',
'info_dict': {
'id': 'cffc0a00-e73f-11ec-8080-80017d29f26e',
'ext': 'mp4',
'title': 'GOOOOD MORNINNNNN #highlights',
'timestamp': 1654702180,
'upload_date': '20220608',
'uploader': 'RahJON Wicc',
'uploader_id': 'TsuSurf',
'duration': 3145,
'age_limit': 17,
'thumbnail': 'https://www.caffeine.tv/broadcasts/776b6f84-9cd5-42e3-af1d-4a776eeed697/replay/lobby.jpg',
'comment_count': int,
'view_count': int,
'like_count': int,
'tags': ['highlights', 'battlerap'],
},
'params': {
'skip_download': 'm3u8',
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
json_data = self._download_json(
f'https://api.caffeine.tv/social/public/activity/{video_id}', video_id)
broadcast_info = traverse_obj(json_data, ('broadcast_info', {dict})) or {}
video_url = broadcast_info['video_url']
ext = determine_ext(video_url)
if ext == 'm3u8':
formats = self._extract_m3u8_formats(video_url, video_id, 'mp4')
else:
formats = [{'url': video_url}]
return {
'id': video_id,
'formats': formats,
**traverse_obj(json_data, {
'like_count': ('like_count', {int_or_none}),
'view_count': ('view_count', {int_or_none}),
'comment_count': ('comment_count', {int_or_none}),
'tags': ('tags', ..., {str}, {lambda x: x or None}),
'uploader': ('user', 'name', {str}),
'uploader_id': (((None, 'user'), 'username'), {str}, any),
'is_live': ('is_live', {bool}),
}),
**traverse_obj(broadcast_info, {
'title': ('broadcast_title', {str}),
'duration': ('content_duration', {int_or_none}),
'timestamp': ('broadcast_start_time', {parse_iso8601}),
'thumbnail': ('preview_image_path', {lambda x: urljoin(url, x)}),
}),
'age_limit': {
# assume Apple Store ratings: https://en.wikipedia.org/wiki/Mobile_software_content_rating_system
'FOUR_PLUS': 0,
'NINE_PLUS': 9,
'TWELVE_PLUS': 12,
'SEVENTEEN_PLUS': 17,
}.get(broadcast_info.get('content_rating'), 17),
}

View file

@ -3384,23 +3384,16 @@ def manifest_url(manifest):
return formats return formats
def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json): def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
mobj = re.search( return self._search_json(
r'''(?s)jwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?!</script>).*?\.\s*setup\s*\(\s*(?P<options>(?:\([^)]*\)|[^)])+)\s*\)''', r'''(?<!-)\bjwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?:(?!</script>).)*?\.\s*(?:setup\s*\(|(?P<load>load)\s*\(\s*\[)''',
webpage) webpage, 'JWPlayer data', video_id,
if mobj: # must be a {...} or sequence, ending
try: contains_pattern=r'\{(?s:.*)}(?(load)(?:\s*,\s*\{(?s:.*)})*)', end_pattern=r'(?(load)\]|\))',
jwplayer_data = self._parse_json(mobj.group('options'), transform_source=transform_source, default=None)
video_id=video_id,
transform_source=transform_source)
except ExtractorError:
pass
else:
if isinstance(jwplayer_data, dict):
return jwplayer_data
def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs): def _extract_jwplayer_data(self, webpage, video_id, *args, transform_source=js_to_json, **kwargs):
jwplayer_data = self._find_jwplayer_data( jwplayer_data = self._find_jwplayer_data(
webpage, video_id, transform_source=js_to_json) webpage, video_id, transform_source=transform_source)
return self._parse_jwplayer_data( return self._parse_jwplayer_data(
jwplayer_data, video_id, *args, **kwargs) jwplayer_data, video_id, *args, **kwargs)
@ -3432,16 +3425,8 @@ def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url) mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
subtitles = {} subtitles = {}
tracks = video_data.get('tracks') for track in traverse_obj(video_data, (
if tracks and isinstance(tracks, list): 'tracks', lambda _, v: v['kind'].lower() in ('captions', 'subtitles'))):
for track in tracks:
if not isinstance(track, dict):
continue
track_kind = track.get('kind')
if not track_kind or not isinstance(track_kind, str):
continue
if track_kind.lower() not in ('captions', 'subtitles'):
continue
track_url = urljoin(base_url, track.get('file')) track_url = urljoin(base_url, track.get('file'))
if not track_url: if not track_url:
continue continue

107
yt_dlp/extractor/gbnews.py Normal file
View file

@ -0,0 +1,107 @@
import functools
from .common import InfoExtractor
from ..utils import (
ExtractorError,
extract_attributes,
get_elements_html_by_class,
url_or_none,
)
from ..utils.traversal import traverse_obj
class GBNewsIE(InfoExtractor):
IE_DESC = 'GB News clips, features and live streams'
_VALID_URL = r'https?://(?:www\.)?gbnews\.(?:uk|com)/(?:\w+/)?(?P<id>[^#?]+)'
_PLATFORM = 'safari'
_SSMP_URL = 'https://mm-v2.simplestream.com/ssmp/api.php'
_TESTS = [{
'url': 'https://www.gbnews.com/news/bbc-claudine-gay-harvard-university-antisemitism-row',
'info_dict': {
'id': '52264136',
'ext': 'mp4',
'thumbnail': r're:https?://www\.gbnews\.\w+/.+\.(?:jpe?g|png|webp)',
'display_id': 'bbc-claudine-gay-harvard-university-antisemitism-row',
'description': 'The post was criticised by former employers of the broadcaster',
'title': 'BBC deletes post after furious backlash over headline downplaying antisemitism',
},
}, {
'url': 'https://www.gbnews.com/royal/prince-harry-in-love-with-kate-meghan-markle-jealous-royal',
'info_dict': {
'id': '52328390',
'ext': 'mp4',
'thumbnail': r're:https?://www\.gbnews\.\w+/.+\.(?:jpe?g|png|webp)',
'display_id': 'prince-harry-in-love-with-kate-meghan-markle-jealous-royal',
'description': 'Ingrid Seward has published 17 books documenting the highs and lows of the Royal Family',
'title': 'Royal author claims Prince Harry was \'in love\' with Kate - Meghan was \'jealous\'',
}
}, {
'url': 'https://www.gbnews.uk/watchlive',
'info_dict': {
'id': '1069',
'ext': 'mp4',
'thumbnail': r're:https?://www\.gbnews\.\w+/.+\.(?:jpe?g|png|webp)',
'display_id': 'watchlive',
'live_status': 'is_live',
'title': r're:^GB News Live',
},
'params': {'skip_download': 'm3u8'},
}]
@functools.lru_cache
def _get_ss_endpoint(self, data_id, data_env):
if not data_id:
data_id = 'GB003'
if not data_env:
data_env = 'production'
json_data = self._download_json(
self._SSMP_URL, None, 'Downloading Simplestream JSON metadata', query={
'id': data_id,
'env': data_env,
})
meta_url = traverse_obj(json_data, ('response', 'api_hostname', {url_or_none}))
if not meta_url:
raise ExtractorError('No API host found')
return meta_url
def _real_extract(self, url):
display_id = self._match_id(url).rpartition('/')[2]
webpage = self._download_webpage(url, display_id)
video_data = None
elements = get_elements_html_by_class('simplestream', webpage)
for html_tag in elements:
attributes = extract_attributes(html_tag)
if 'sidebar' not in (attributes.get('class') or ''):
video_data = attributes
if not video_data:
raise ExtractorError('Could not find video element', expected=True)
endpoint_url = self._get_ss_endpoint(video_data.get('data-id'), video_data.get('data-env'))
uvid = video_data['data-uvid']
video_type = video_data.get('data-type')
if not video_type or video_type == 'vod':
video_type = 'show'
stream_data = self._download_json(
f'{endpoint_url}/api/{video_type}/stream/{uvid}',
uvid, 'Downloading stream JSON', query={
'key': video_data.get('data-key'),
'platform': self._PLATFORM,
})
if traverse_obj(stream_data, 'drm'):
self.report_drm(uvid)
return {
'id': uvid,
'display_id': display_id,
'title': self._og_search_title(webpage, default=None),
'description': self._og_search_description(webpage, default=None),
'formats': self._extract_m3u8_formats(traverse_obj(stream_data, (
'response', 'stream', {url_or_none})), uvid, 'mp4'),
'thumbnail': self._og_search_thumbnail(webpage, default=None),
'is_live': video_type == 'live',
}

View file

@ -1,19 +1,27 @@
import itertools
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
ExtractorError,
clean_html,
extract_attributes, extract_attributes,
get_element_by_class,
get_element_by_id,
get_elements_html_by_class,
int_or_none, int_or_none,
merge_dicts, merge_dicts,
str_to_int, parse_count,
parse_qs,
traverse_obj, traverse_obj,
unified_strdate, unified_strdate,
url_or_none, url_or_none,
urljoin,
) )
class YouPornIE(InfoExtractor): class YouPornIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?' _VALID_URL = r'https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?/?(?:[#?]|$)'
_EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)'] _EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)']
_TESTS = [{ _TESTS = [{
'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/', 'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
@ -34,7 +42,7 @@ class YouPornIE(InfoExtractor):
'tags': list, 'tags': list,
'age_limit': 18, 'age_limit': 18,
}, },
'skip': 'This video has been disabled', 'skip': 'This video has been deactivated',
}, { }, {
# Unknown uploader # Unknown uploader
'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4', 'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
@ -72,7 +80,6 @@ class YouPornIE(InfoExtractor):
'id': '16290308', 'id': '16290308',
'age_limit': 18, 'age_limit': 18,
'categories': [], 'categories': [],
'description': str, # TODO: detect/remove SEO spam description in ytdl backport
'display_id': 'tinderspecial-trailer1', 'display_id': 'tinderspecial-trailer1',
'duration': 298.0, 'duration': 298.0,
'ext': 'mp4', 'ext': 'mp4',
@ -90,7 +97,17 @@ def _real_extract(self, url):
video_id, display_id = self._match_valid_url(url).group('id', 'display_id') video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
self._set_cookie('.youporn.com', 'age_verified', '1') self._set_cookie('.youporn.com', 'age_verified', '1')
webpage = self._download_webpage(f'https://www.youporn.com/watch/{video_id}', video_id) webpage = self._download_webpage(f'https://www.youporn.com/watch/{video_id}', video_id)
definitions = self._search_json(r'\bplayervars\s*:', webpage, 'player vars', video_id)['mediaDefinitions']
watchable = self._search_regex(
r'''(<div\s[^>]*\bid\s*=\s*('|")?watch-container(?(2)\2|(?!-)\b)[^>]*>)''',
webpage, 'watchability', default=None)
if not watchable:
msg = re.split(r'\s{2}', clean_html(get_element_by_id('mainContent', webpage)) or '')[0]
raise ExtractorError(
f'{self.IE_NAME} says: {msg}' if msg else 'Video unavailable', expected=True)
player_vars = self._search_json(r'\bplayervars\s*:', webpage, 'player vars', video_id)
definitions = player_vars['mediaDefinitions']
def get_format_data(data, stream_type): def get_format_data(data, stream_type):
info_url = traverse_obj(data, (lambda _, v: v['format'] == stream_type, 'videoUrl', {url_or_none}, any)) info_url = traverse_obj(data, (lambda _, v: v['format'] == stream_type, 'videoUrl', {url_or_none}, any))
@ -143,6 +160,8 @@ def get_format_data(data, stream_type):
thumbnail = self._search_regex( thumbnail = self._search_regex(
r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1', r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
webpage, 'thumbnail', fatal=False, group='thumbnail') webpage, 'thumbnail', fatal=False, group='thumbnail')
duration = traverse_obj(player_vars, ('duration', {int_or_none}))
if duration is None:
duration = int_or_none(self._html_search_meta( duration = int_or_none(self._html_search_meta(
'video:duration', webpage, 'duration', fatal=False)) 'video:duration', webpage, 'duration', fatal=False))
@ -160,11 +179,11 @@ def get_format_data(data, stream_type):
view_count = None view_count = None
views = self._search_regex( views = self._search_regex(
r'(<div[^>]+\bclass=["\']js_videoInfoViews["\']>)', webpage, r'(<div [^>]*\bdata-value\s*=[^>]+>)\s*<label>Views:</label>',
'views', default=None) webpage, 'views', default=None)
if views: if views:
view_count = str_to_int(extract_attributes(views).get('data-value')) view_count = parse_count(extract_attributes(views).get('data-value'))
comment_count = str_to_int(self._search_regex( comment_count = parse_count(self._search_regex(
r'>All [Cc]omments? \(([\d,.]+)\)', r'>All [Cc]omments? \(([\d,.]+)\)',
webpage, 'comment count', default=None)) webpage, 'comment count', default=None))
@ -182,7 +201,8 @@ def extract_tag_box(regex, title):
data = self._search_json_ld(webpage, video_id, expected_type='VideoObject', fatal=False) data = self._search_json_ld(webpage, video_id, expected_type='VideoObject', fatal=False)
data.pop('url', None) data.pop('url', None)
return merge_dicts(data, {
result = merge_dicts(data, {
'id': video_id, 'id': video_id,
'display_id': display_id, 'display_id': display_id,
'title': title, 'title': title,
@ -198,3 +218,350 @@ def extract_tag_box(regex, title):
'age_limit': age_limit, 'age_limit': age_limit,
'formats': formats, 'formats': formats,
}) })
# Remove SEO spam "description"
description = result.get('description')
if description and description.startswith(f'Watch {result.get("title")} online'):
del result['description']
return result
class YouPornListBase(InfoExtractor):
def _get_next_url(self, url, pl_id, html):
return urljoin(url, self._search_regex(
r'''<a [^>]*?\bhref\s*=\s*("|')(?P<url>(?:(?!\1)[^>])+)\1''',
get_element_by_id('next', html) or '', 'next page',
group='url', default=None))
@classmethod
def _get_title_from_slug(cls, title_slug):
return re.sub(r'[_-]', ' ', title_slug)
def _entries(self, url, pl_id, html=None, page_num=None):
start = page_num or 1
for page in itertools.count(start):
if not html:
html = self._download_webpage(
url, pl_id, note=f'Downloading page {page}', fatal=page == start)
if not html:
return
for element in get_elements_html_by_class('video-title', html):
if video_url := traverse_obj(element, ({extract_attributes}, 'href', {lambda x: urljoin(url, x)})):
yield self.url_result(video_url)
if page_num is not None:
return
next_url = self._get_next_url(url, pl_id, html)
if not next_url or next_url == url:
return
url = next_url
html = None
def _real_extract(self, url, html=None):
m_dict = self._match_valid_url(url).groupdict()
pl_id, page_type, sort = (m_dict.get(k) for k in ('id', 'type', 'sort'))
qs = {k: v[-1] for k, v in parse_qs(url).items() if v}
base_id = pl_id or 'YouPorn'
title = self._get_title_from_slug(base_id)
if page_type:
title = f'{page_type.capitalize()} {title}'
base_id = [base_id.lower()]
if sort is None:
title += ' videos'
else:
title = f'{title} videos by {re.sub(r"[_-]", " ", sort)}'
base_id.append(sort)
if qs:
filters = list(map('='.join, sorted(qs.items())))
title += f' ({",".join(filters)})'
base_id.extend(filters)
pl_id = '/'.join(base_id)
return self.playlist_result(
self._entries(url, pl_id, html=html, page_num=int_or_none(qs.get('page'))),
playlist_id=pl_id, playlist_title=title)
class YouPornCategoryIE(YouPornListBase):
IE_DESC = 'YouPorn category, with sorting, filtering and pagination'
_VALID_URL = r'''(?x)
https?://(?:www\.)?youporn\.com/
(?P<type>category)/(?P<id>[^/?#&]+)
(?:/(?P<sort>popular|views|rating|time|duration))?/?(?:[#?]|$)
'''
_TESTS = [{
'note': 'Full list with pagination',
'url': 'https://www.youporn.com/category/popular-with-women/popular/',
'info_dict': {
'id': 'popular-with-women/popular',
'title': 'Category popular with women videos by popular',
},
'playlist_mincount': 39,
}, {
'note': 'Filtered paginated list with single page result',
'url': 'https://www.youporn.com/category/popular-with-women/duration/?min_minutes=10',
'info_dict': {
'id': 'popular-with-women/duration/min_minutes=10',
'title': 'Category popular with women videos by duration (min_minutes=10)',
},
'playlist_mincount': 2,
# 'playlist_maxcount': 30,
}, {
'note': 'Single page of full list',
'url': 'https://www.youporn.com/category/popular-with-women/popular?page=1',
'info_dict': {
'id': 'popular-with-women/popular/page=1',
'title': 'Category popular with women videos by popular (page=1)',
},
'playlist_count': 36,
}]
class YouPornChannelIE(YouPornListBase):
IE_DESC = 'YouPorn channel, with sorting and pagination'
_VALID_URL = r'''(?x)
https?://(?:www\.)?youporn\.com/
(?P<type>channel)/(?P<id>[^/?#&]+)
(?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
'''
_TESTS = [{
'note': 'Full list with pagination',
'url': 'https://www.youporn.com/channel/x-feeds/',
'info_dict': {
'id': 'x-feeds',
'title': 'Channel X-Feeds videos',
},
'playlist_mincount': 37,
}, {
'note': 'Single page of full list (no filters here)',
'url': 'https://www.youporn.com/channel/x-feeds/duration?page=1',
'info_dict': {
'id': 'x-feeds/duration/page=1',
'title': 'Channel X-Feeds videos by duration (page=1)',
},
'playlist_count': 24,
}]
@staticmethod
def _get_title_from_slug(title_slug):
return re.sub(r'_', ' ', title_slug).title()
class YouPornCollectionIE(YouPornListBase):
IE_DESC = 'YouPorn collection (user playlist), with sorting and pagination'
_VALID_URL = r'''(?x)
https?://(?:www\.)?youporn\.com/
(?P<type>collection)s/videos/(?P<id>\d+)
(?:/(?P<sort>rating|views|time|duration))?/?(?:[#?]|$)
'''
_TESTS = [{
'note': 'Full list with pagination',
'url': 'https://www.youporn.com/collections/videos/33044251/',
'info_dict': {
'id': '33044251',
'title': 'Collection Sexy Lips videos',
'uploader': 'ph-littlewillyb',
},
'playlist_mincount': 50,
}, {
'note': 'Single page of full list (no filters here)',
'url': 'https://www.youporn.com/collections/videos/33044251/time?page=1',
'info_dict': {
'id': '33044251/time/page=1',
'title': 'Collection Sexy Lips videos by time (page=1)',
'uploader': 'ph-littlewillyb',
},
'playlist_count': 20,
}]
def _real_extract(self, url):
pl_id = self._match_id(url)
html = self._download_webpage(url, pl_id)
playlist = super()._real_extract(url, html=html)
infos = re.sub(r'\s+', ' ', clean_html(get_element_by_class(
'collection-infos', html)) or '')
title, uploader = self._search_regex(
r'^\s*Collection: (?P<title>.+?) \d+ VIDEOS \d+ VIEWS \d+ days LAST UPDATED From: (?P<uploader>[\w_-]+)',
infos, 'title/uploader', group=('title', 'uploader'), default=(None, None))
if title:
playlist.update({
'title': playlist['title'].replace(playlist['id'].split('/')[0], title),
'uploader': uploader,
})
return playlist
class YouPornTagIE(YouPornListBase):
IE_DESC = 'YouPorn tag (porntags), with sorting, filtering and pagination'
_VALID_URL = r'''(?x)
https?://(?:www\.)?youporn\.com/
porn(?P<type>tag)s/(?P<id>[^/?#&]+)
(?:/(?P<sort>views|rating|time|duration))?/?(?:[#?]|$)
'''
_TESTS = [{
'note': 'Full list with pagination',
'url': 'https://www.youporn.com/porntags/austrian',
'info_dict': {
'id': 'austrian',
'title': 'Tag austrian videos',
},
'playlist_mincount': 33,
'expected_warnings': ['YouPorn tag pages are not correctly cached'],
}, {
'note': 'Filtered paginated list with single page result',
'url': 'https://www.youporn.com/porntags/austrian/duration/?min_minutes=10',
'info_dict': {
'id': 'austrian/duration/min_minutes=10',
'title': 'Tag austrian videos by duration (min_minutes=10)',
},
'playlist_mincount': 10,
# number of videos per page is (row x col) 2x3 + 6x4 + 2, or + 3,
# or more, varying with number of ads; let's set max as 9x4
# NB col 1 may not be shown in non-JS page with site CSS and zoom 100%
# 'playlist_maxcount': 32,
'expected_warnings': ['YouPorn tag pages are not correctly cached'],
}, {
'note': 'Single page of full list',
'url': 'https://www.youporn.com/porntags/austrian/?page=1',
'info_dict': {
'id': 'austrian/page=1',
'title': 'Tag austrian videos (page=1)',
},
'playlist_mincount': 32,
# 'playlist_maxcount': 34,
'expected_warnings': ['YouPorn tag pages are not correctly cached'],
}]
def _real_extract(self, url):
self.report_warning(
'YouPorn tag pages are not correctly cached and '
'often return incorrect results', only_once=True)
return super()._real_extract(url)
class YouPornStarIE(YouPornListBase):
IE_DESC = 'YouPorn Pornstar, with description, sorting and pagination'
_VALID_URL = r'''(?x)
https?://(?:www\.)?youporn\.com/
(?P<type>pornstar)/(?P<id>[^/?#&]+)
(?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
'''
_TESTS = [{
'note': 'Full list with pagination',
'url': 'https://www.youporn.com/pornstar/daynia/',
'info_dict': {
'id': 'daynia',
'title': 'Pornstar Daynia videos',
'description': r're:Daynia Rank \d+ Videos \d+ Views [\d,.]+ .+ Subscribers \d+',
},
'playlist_mincount': 40,
}, {
'note': 'Single page of full list (no filters here)',
'url': 'https://www.youporn.com/pornstar/daynia/?page=1',
'info_dict': {
'id': 'daynia/page=1',
'title': 'Pornstar Daynia videos (page=1)',
'description': 're:.{180,}',
},
'playlist_count': 26,
}]
@staticmethod
def _get_title_from_slug(title_slug):
return re.sub(r'_', ' ', title_slug).title()
def _real_extract(self, url):
pl_id = self._match_id(url)
html = self._download_webpage(url, pl_id)
playlist = super()._real_extract(url, html=html)
INFO_ELEMENT_RE = r'''(?x)
<div [^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?pornstar-info-wrapper(?:\s+[\w$-]+|\s)*\1[^>]*>
(?P<info>[\s\S]+?)(?:</div>\s*){6,}
'''
if infos := self._search_regex(INFO_ELEMENT_RE, html, 'infos', group='info', default=''):
infos = re.sub(
r'(?:\s*nl=nl)+\s*', ' ',
re.sub(r'(?u)\s+', ' ', clean_html(re.sub('\n', 'nl=nl', infos)))).replace('ribe Subsc', '')
return {
**playlist,
'description': infos.strip() or None,
}
class YouPornVideosIE(YouPornListBase):
IE_DESC = 'YouPorn video (browse) playlists, with sorting, filtering and pagination'
_VALID_URL = r'''(?x)
https?://(?:www\.)?youporn\.com/
(?:(?P<id>browse)/)?
(?P<sort>(?(id)
(?:duration|rating|time|views)|
(?:most_(?:favou?rit|view)ed|recommended|top_rated)?))
(?:[/#?]|$)
'''
_TESTS = [{
'note': 'Full list with pagination (too long for test)',
'url': 'https://www.youporn.com/',
'info_dict': {
'id': 'youporn',
'title': 'YouPorn videos',
},
'only_matching': True,
}, {
'note': 'Full list with pagination (too long for test)',
'url': 'https://www.youporn.com/recommended',
'info_dict': {
'id': 'youporn/recommended',
'title': 'YouPorn videos by recommended',
},
'only_matching': True,
}, {
'note': 'Full list with pagination (too long for test)',
'url': 'https://www.youporn.com/top_rated',
'info_dict': {
'id': 'youporn/top_rated',
'title': 'YouPorn videos by top rated',
},
'only_matching': True,
}, {
'note': 'Full list with pagination (too long for test)',
'url': 'https://www.youporn.com/browse/time',
'info_dict': {
'id': 'browse/time',
'title': 'YouPorn videos by time',
},
'only_matching': True,
}, {
'note': 'Filtered paginated list with single page result',
'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=2',
'info_dict': {
'id': 'youporn/most_favorited/max_minutes=2/res=VR',
'title': 'YouPorn videos by most favorited (max_minutes=2,res=VR)',
},
'playlist_mincount': 10,
# 'playlist_maxcount': 28,
}, {
'note': 'Filtered paginated list with several pages',
'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=5',
'info_dict': {
'id': 'youporn/most_favorited/max_minutes=5/res=VR',
'title': 'YouPorn videos by most favorited (max_minutes=5,res=VR)',
},
'playlist_mincount': 45,
}, {
'note': 'Single page of full list',
'url': 'https://www.youporn.com/browse/time?page=1',
'info_dict': {
'id': 'browse/time/page=1',
'title': 'YouPorn videos by time (page=1)',
},
'playlist_count': 36,
}]
@staticmethod
def _get_title_from_slug(title_slug):
return 'YouPorn' if title_slug == 'browse' else title_slug