From 902784a2a9830f999a7d275b374952fc44bbbc02 Mon Sep 17 00:00:00 2001 From: nixxo Date: Mon, 9 Nov 2020 09:16:37 +0100 Subject: [PATCH] [gedi] added huffingtonpost, added embeds --- youtube_dlc/extractor/extractors.py | 5 +- youtube_dlc/extractor/gedi.py | 89 +++++++++++++++++++++++++++++ youtube_dlc/extractor/generic.py | 7 +++ 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/youtube_dlc/extractor/extractors.py b/youtube_dlc/extractor/extractors.py index bee90d74a..01f69c006 100644 --- a/youtube_dlc/extractor/extractors.py +++ b/youtube_dlc/extractor/extractors.py @@ -414,7 +414,10 @@ from .gaskrank import GaskrankIE from .gazeta import GazetaIE from .gdcvault import GDCVaultIE -from .gedi import GediIE +from .gedi import ( + GediIE, + GediEmbedsIE, +) from .generic import GenericIE from .gfycat import GfycatIE from .giantbomb import GiantBombIE diff --git a/youtube_dlc/extractor/gedi.py b/youtube_dlc/extractor/gedi.py index 5efc8a6e9..f35dfca5a 100644 --- a/youtube_dlc/extractor/gedi.py +++ b/youtube_dlc/extractor/gedi.py @@ -5,6 +5,11 @@ from .common import InfoExtractor from ..compat import compat_str +from ..utils import ( + base_url, + url_basename, + urljoin, +) class GediBaseIE(InfoExtractor): @@ -109,6 +114,7 @@ class GediIE(GediBaseIE): (?: (?:espresso\.)?repubblica |lastampa + |huffingtonpost |ilsecoloxix |iltirreno |messaggeroveneto @@ -175,3 +181,86 @@ class GediIE(GediBaseIE): 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$', }, }] + + +class GediEmbedsIE(GediBaseIE): + _VALID_URL = r'''(?x)https?://video\. + (?: + (?:espresso\.)?repubblica + |lastampa + |huffingtonpost + |ilsecoloxix + |iltirreno + |messaggeroveneto + |ilpiccolo + |gazzettadimantova + |mattinopadova + |laprovinciapavese + |tribunatreviso + |nuovavenezia + |gazzettadimodena + |lanuovaferrara + |corrierealpi + |lasentinella + ) + (?:\.gelocal)?\.it/embed/.+?/(?P[\d/]+)(?:\?|\&|$)''' + _TESTS = [{ + 'url': 'https://video.huffingtonpost.it/embed/politica/cotticelli-non-so-cosa-mi-sia-successo-sto-cercando-di-capire-se-ho-avuto-un-malore/29312/29276?responsive=true&el=video971040871621586700', + 'md5': '0391c2c83c6506581003aaf0255889c0', + 'info_dict': { + 'id': '14772/14870', + 'ext': 'mp4', + 'title': 'Festival EMERGENCY, Villa: «La buona informazione aiuta la salute» (14772-14870)', + 'description': 'md5:2bce954d278248f3c950be355b7c2226', + 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$', + }, + }] + + @staticmethod + def _sanitize_urls(urls): + # add protocol if missing + for i, e in enumerate(urls): + if e.startswith('//'): + urls[i] = 'https:%s' % e + # clean iframes urls + for i, e in enumerate(urls): + urls[i] = urljoin(base_url(e), url_basename(e)) + return urls + + @staticmethod + def _extract_urls(webpage): + entries = [ + mobj.group('url') + for mobj in re.finditer(r'''(?x) + (?: + data-frame-src=| + https?://video\. + (?: + (?:espresso\.)?repubblica + |lastampa + |huffingtonpost + |ilsecoloxix + |iltirreno + |messaggeroveneto + |ilpiccolo + |gazzettadimantova + |mattinopadova + |laprovinciapavese + |tribunatreviso + |nuovavenezia + |gazzettadimodena + |lanuovaferrara + |corrierealpi + |lasentinella + ) + (?:\.gelocal)?\.it/embed/.+?) + \1''', webpage)] + return GediEmbedsIE._sanitize_urls(entries) + + @staticmethod + def _extract_url(webpage): + urls = GediEmbedsIE._extract_urls(webpage) + return urls[0] if urls else None diff --git a/youtube_dlc/extractor/generic.py b/youtube_dlc/extractor/generic.py index aba06b328..3fab929a8 100644 --- a/youtube_dlc/extractor/generic.py +++ b/youtube_dlc/extractor/generic.py @@ -119,6 +119,7 @@ from .zype import ZypeIE from .odnoklassniki import OdnoklassnikiIE from .kinja import KinjaEmbedIE +from .gedi import GediEmbedsIE class GenericIE(InfoExtractor): @@ -3213,6 +3214,12 @@ def _real_extract(self, url): return self.playlist_from_matches( zype_urls, video_id, video_title, ie=ZypeIE.ie_key()) + # Look for RCS media group embeds + gedi_urls = GediEmbedsIE._extract_urls(webpage) + if gedi_urls: + return self.playlist_from_matches( + gedi_urls, video_id, video_title, ie=GediEmbedsIE.ie_key()) + # Look for HTML5 media entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls') if entries: