From ee45625290c678c53b7ee86d2adba40847b79573 Mon Sep 17 00:00:00 2001 From: MaxReimann Date: Sun, 21 Dec 2014 11:01:28 +0100 Subject: [PATCH 1/4] Add extractor for teletask --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/teletask.py | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 youtube_dl/extractor/teletask.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 76f13bf52..073ac1fab 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -405,6 +405,7 @@ from .telebruxelles import TeleBruxellesIE from .telecinco import TelecincoIE from .telemb import TeleMBIE +from .teletask import TeleTaskIE from .tenplay import TenPlayIE from .testurl import TestURLIE from .tf1 import TF1IE diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py new file mode 100644 index 000000000..1c42ac6f4 --- /dev/null +++ b/youtube_dl/extractor/teletask.py @@ -0,0 +1,59 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re +import datetime + +from .common import InfoExtractor + + +class TeleTaskIE(InfoExtractor): + _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P[0-9]+)/' + _TEST = { + 'url': 'http://www.tele-task.de/archive/video/html5/26168/', + 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', + 'info_dict': { + 'id': '26168', + 'ext': 'mp4', + 'title': 'Duplicate Detection', + 'thumbnail': 're:^https?://.*\.jpg$', + 'date': '20141218', + # TODO more properties, either as: + # * A value + # * MD5 checksum; start the string with md5: + # * A regular expression; start the string with re: + # * Any Python type (for example int or float) + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + lecture_url = self._html_search_regex( + r'href="([^"]+)" itemprop="name">', webpage, 'title') + lecture_id = re.search("([0-9]+)/",lecture_url).group(1) + overview_page = self._download_webpage("http://www.tele-task.de" + lecture_url, + lecture_id) + + title = self._html_search_regex( + r'itemprop="name">([^"]+)', webpage, 'title') + url = self._html_search_regex( + r'class="speaker".*?src="([^"]+)"', webpage, 'video_url', flags=re.DOTALL) + description = self._html_search_regex( + r'Description of the series:

([^"]+)', overview_page, + 'description',flags=re.DOTALL) + + date = self._html_search_regex( + r'Date:([^"]+)', webpage, 'date') + date = datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y%m%d') + + + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'url': url, + 'upload_date': date, + } \ No newline at end of file From 3f7421b71b0d7da917a92b6f37985bc26c5dc11d Mon Sep 17 00:00:00 2001 From: MaxReimann Date: Sun, 21 Dec 2014 11:13:59 +0100 Subject: [PATCH 2/4] fix test and remove lengthy description --- youtube_dl/extractor/teletask.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py index 1c42ac6f4..6920b871e 100644 --- a/youtube_dl/extractor/teletask.py +++ b/youtube_dl/extractor/teletask.py @@ -11,18 +11,12 @@ class TeleTaskIE(InfoExtractor): _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P[0-9]+)/' _TEST = { 'url': 'http://www.tele-task.de/archive/video/html5/26168/', - 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', + 'md5': '290ef69fb2792e481169c3958dbfbd57', 'info_dict': { 'id': '26168', 'ext': 'mp4', 'title': 'Duplicate Detection', - 'thumbnail': 're:^https?://.*\.jpg$', - 'date': '20141218', - # TODO more properties, either as: - # * A value - # * MD5 checksum; start the string with md5: - # * A regular expression; start the string with re: - # * Any Python type (for example int or float) + 'upload_date': '20141218', } } @@ -30,19 +24,10 @@ def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - lecture_url = self._html_search_regex( - r'href="([^"]+)" itemprop="name">', webpage, 'title') - lecture_id = re.search("([0-9]+)/",lecture_url).group(1) - overview_page = self._download_webpage("http://www.tele-task.de" + lecture_url, - lecture_id) - title = self._html_search_regex( r'itemprop="name">([^"]+)', webpage, 'title') url = self._html_search_regex( r'class="speaker".*?src="([^"]+)"', webpage, 'video_url', flags=re.DOTALL) - description = self._html_search_regex( - r'Description of the series:

([^"]+)', overview_page, - 'description',flags=re.DOTALL) date = self._html_search_regex( r'Date:([^"]+)', webpage, 'date') @@ -53,7 +38,6 @@ def _real_extract(self, url): return { 'id': video_id, 'title': title, - 'description': description, 'url': url, 'upload_date': date, } \ No newline at end of file From 33b53b60212ca28f589616f1acb650b323a18f11 Mon Sep 17 00:00:00 2001 From: MaxReimann Date: Sun, 21 Dec 2014 12:26:47 +0100 Subject: [PATCH 3/4] [teletask] Add new extractor --- youtube_dl/extractor/teletask.py | 62 ++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py index 6920b871e..aa5535042 100644 --- a/youtube_dl/extractor/teletask.py +++ b/youtube_dl/extractor/teletask.py @@ -1,7 +1,6 @@ # coding: utf-8 from __future__ import unicode_literals - -import re +import re import datetime from .common import InfoExtractor @@ -10,34 +9,59 @@ class TeleTaskIE(InfoExtractor): _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P[0-9]+)/' _TEST = { - 'url': 'http://www.tele-task.de/archive/video/html5/26168/', - 'md5': '290ef69fb2792e481169c3958dbfbd57', + 'url': 'http://www.tele-task.de/archive/video/html5/26168/', 'info_dict': { - 'id': '26168', - 'ext': 'mp4', 'title': 'Duplicate Detection', - 'upload_date': '20141218', - } + }, + 'playlist': [{ + 'md5': '290ef69fb2792e481169c3958dbfbd57', + 'info_dict': { + 'title': 'Duplicate Detection', + 'upload_date': '20141218', + 'id': 'speaker_26168', + 'ext': 'mp4', + } + }, + { + 'md5': 'e1e7218c5f0e4790015a437fcf6c71b4', + 'info_dict': { + 'title': 'Duplicate Detection', + 'upload_date': '20141218', + 'id': 'slides_26168', + 'ext': 'mp4', + } + }] } def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - + lecture_id = self._match_id(url) + webpage = self._download_webpage(url, lecture_id) + title = self._html_search_regex( r'itemprop="name">([^"]+)', webpage, 'title') - url = self._html_search_regex( - r'class="speaker".*?src="([^"]+)"', webpage, 'video_url', flags=re.DOTALL) - + url_speaker = self._html_search_regex( + r'class="speaker".*?src="([^"]+)"', webpage, 'video_url_speaker', flags=re.DOTALL) + url_slides = self._html_search_regex( + r'class="slides".*?src="([^"]+)"', webpage, 'video_url_slides', flags=re.DOTALL) date = self._html_search_regex( r'Date:([^"]+)', webpage, 'date') date = datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y%m%d') - + entries = [{ + 'title': title, + 'upload_date': date, + 'id': "speaker_"+lecture_id, + 'url': url_speaker, + }, + { + 'title': title, + 'upload_date': date, + 'id': "slides_"+lecture_id, + 'url': url_slides}] return { - 'id': video_id, + '_type': "playlist", + 'id': lecture_id, 'title': title, - 'url': url, - 'upload_date': date, - } \ No newline at end of file + 'entries': entries, + } From 885e4384a1ad8b12037810824d15e9436134a148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 25 Dec 2014 23:26:57 +0600 Subject: [PATCH 4/4] [teletask] Simplify --- youtube_dl/extractor/teletask.py | 50 ++++++++++++-------------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py index aa5535042..e54145105 100644 --- a/youtube_dl/extractor/teletask.py +++ b/youtube_dl/extractor/teletask.py @@ -1,13 +1,13 @@ -# coding: utf-8 from __future__ import unicode_literals + import re -import datetime from .common import InfoExtractor +from ..utils import unified_strdate class TeleTaskIE(InfoExtractor): - _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P[0-9]+)/' + _VALID_URL = r'https?://(?:www\.)?tele-task\.de/archive/video/html5/(?P[0-9]+)' _TEST = { 'url': 'http://www.tele-task.de/archive/video/html5/26168/', 'info_dict': { @@ -16,52 +16,38 @@ class TeleTaskIE(InfoExtractor): 'playlist': [{ 'md5': '290ef69fb2792e481169c3958dbfbd57', 'info_dict': { + 'id': '26168-speaker', + 'ext': 'mp4', 'title': 'Duplicate Detection', 'upload_date': '20141218', - 'id': 'speaker_26168', - 'ext': 'mp4', } - }, - { + }, { 'md5': 'e1e7218c5f0e4790015a437fcf6c71b4', 'info_dict': { + 'id': '26168-slides', + 'ext': 'mp4', 'title': 'Duplicate Detection', 'upload_date': '20141218', - 'id': 'slides_26168', - 'ext': 'mp4', } }] } def _real_extract(self, url): lecture_id = self._match_id(url) + webpage = self._download_webpage(url, lecture_id) title = self._html_search_regex( - r'itemprop="name">([^"]+)', webpage, 'title') - url_speaker = self._html_search_regex( - r'class="speaker".*?src="([^"]+)"', webpage, 'video_url_speaker', flags=re.DOTALL) - url_slides = self._html_search_regex( - r'class="slides".*?src="([^"]+)"', webpage, 'video_url_slides', flags=re.DOTALL) - date = self._html_search_regex( - r'Date:([^"]+)', webpage, 'date') - date = datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y%m%d') + r'itemprop="name">([^<]+)', webpage, 'title') + upload_date = unified_strdate(self._html_search_regex( + r'Date:([^<]+)', webpage, 'date', fatal=False)) entries = [{ + 'id': '%s-%s' % (lecture_id, format_id), + 'url': video_url, 'title': title, - 'upload_date': date, - 'id': "speaker_"+lecture_id, - 'url': url_speaker, - }, - { - 'title': title, - 'upload_date': date, - 'id': "slides_"+lecture_id, - 'url': url_slides}] + 'upload_date': upload_date, + } for format_id, video_url in re.findall( + r'