Merge branch 'master' into extract_info_rewrite

This commit is contained in:
Jaime Marquínez Ferrándiz 2013-04-05 12:39:51 +02:00
commit 14294236bf
11 changed files with 550 additions and 218 deletions

2
.gitignore vendored
View File

@ -17,4 +17,4 @@ youtube-dl.tar.gz
.coverage .coverage
cover/ cover/
updates_key.pem updates_key.pem
*.egg-info *.egg-info

View File

@ -18,7 +18,7 @@ # OPTIONS
--version print program version and exit --version print program version and exit
-U, --update update this program to latest version -U, --update update this program to latest version
-i, --ignore-errors continue on download errors -i, --ignore-errors continue on download errors
-r, --rate-limit LIMIT download rate limit (e.g. 50k or 44.6m) -r, --rate-limit LIMIT maximum download rate (e.g. 50k or 44.6m)
-R, --retries RETRIES number of retries (default is 10) -R, --retries RETRIES number of retries (default is 10)
--buffer-size SIZE size of download buffer (e.g. 1024 or 16k) (default --buffer-size SIZE size of download buffer (e.g. 1024 or 16k) (default
is 1024) is 1024)
@ -97,10 +97,16 @@ ## Video Format Options:
requested requested
--max-quality FORMAT highest quality format to download --max-quality FORMAT highest quality format to download
-F, --list-formats list all available formats (currently youtube only) -F, --list-formats list all available formats (currently youtube only)
--write-srt write video closed captions to a .srt file --write-sub write subtitle file (currently youtube only)
--only-sub downloads only the subtitles (no video)
--all-subs downloads all the available subtitles of the video
(currently youtube only) (currently youtube only)
--srt-lang LANG language of the closed captions to download --list-subs lists all available subtitles for the video
(optional) use IETF language tags like 'en' (currently youtube only)
--sub-format LANG subtitle format [srt/sbv] (default=srt) (currently
youtube only)
--sub-lang LANG language of the subtitles to download (optional)
use IETF language tags like 'en'
## Authentication Options: ## Authentication Options:
-u, --username USERNAME account username -u, --username USERNAME account username

View File

@ -0,0 +1,57 @@
#!/usr/bin/env python3
import datetime
import textwrap
import json
atom_template=textwrap.dedent("""\
<?xml version='1.0' encoding='utf-8'?>
<atom:feed xmlns:atom="http://www.w3.org/2005/Atom">
<atom:title>youtube-dl releases</atom:title>
<atom:id>youtube-dl-updates-feed</atom:id>
<atom:updated>@TIMESTAMP@</atom:updated>
@ENTRIES@
</atom:feed>""")
entry_template=textwrap.dedent("""
<atom:entry>
<atom:id>youtube-dl-@VERSION@</atom:id>
<atom:title>New version @VERSION@</atom:title>
<atom:link href="http://rg3.github.com/youtube-dl" />
<atom:content type="xhtml">
<div xmlns="http://www.w3.org/1999/xhtml">
Downloads available at <a href="http://youtube-dl.org/downloads/@VERSION@/">http://youtube-dl.org/downloads/@VERSION@/</a>
</div>
</atom:content>
<atom:author>
<atom:name>The youtube-dl maintainers</atom:name>
</atom:author>
<atom:updated>@TIMESTAMP@</atom:updated>
</atom:entry>
""")
now = datetime.datetime.now()
now_iso = now.isoformat()
atom_template = atom_template.replace('@TIMESTAMP@',now_iso)
entries=[]
versions_info = json.load(open('update/versions.json'))
versions = list(versions_info['versions'].keys())
versions.sort()
for v in versions:
entry = entry_template.replace('@TIMESTAMP@',v.replace('.','-'))
entry = entry.replace('@VERSION@',v)
entries.append(entry)
entries_str = textwrap.indent(''.join(entries), '\t')
atom_template = atom_template.replace('@ENTRIES@', entries_str)
with open('update/releases.atom','w',encoding='utf-8') as atom_file:
atom_file.write(atom_template)

View File

@ -69,6 +69,7 @@ ROOT=$(pwd)
ORIGIN_URL=$(git config --get remote.origin.url) ORIGIN_URL=$(git config --get remote.origin.url)
cd build/gh-pages cd build/gh-pages
"$ROOT/devscripts/gh-pages/add-version.py" $version "$ROOT/devscripts/gh-pages/add-version.py" $version
"$ROOT/devscripts/gh-pages/update-feed.py"
"$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem" "$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem"
"$ROOT/devscripts/gh-pages/generate-download.py" "$ROOT/devscripts/gh-pages/generate-download.py"
"$ROOT/devscripts/gh-pages/update-copyright.py" "$ROOT/devscripts/gh-pages/update-copyright.py"

View File

@ -20,6 +20,8 @@
DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json') DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json')
PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json") PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
RETRIES = 3
# General configuration (from __init__, not very elegant...) # General configuration (from __init__, not very elegant...)
jar = compat_cookiejar.CookieJar() jar = compat_cookiejar.CookieJar()
cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar) cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
@ -79,9 +81,8 @@ def test_template(self):
params.update(test_case.get('params', {})) params.update(test_case.get('params', {}))
fd = FileDownloader(params) fd = FileDownloader(params)
fd.add_info_extractor(ie()) for ie in youtube_dl.InfoExtractors.gen_extractors():
for ien in test_case.get('add_ie', []): fd.add_info_extractor(ie)
fd.add_info_extractor(getattr(youtube_dl.InfoExtractors, ien + 'IE')())
finished_hook_called = set() finished_hook_called = set()
def _hook(status): def _hook(status):
if status['status'] == 'finished': if status['status'] == 'finished':
@ -94,7 +95,19 @@ def _hook(status):
_try_rm(tc['file'] + '.part') _try_rm(tc['file'] + '.part')
_try_rm(tc['file'] + '.info.json') _try_rm(tc['file'] + '.info.json')
try: try:
fd.download([test_case['url']]) for retry in range(1, RETRIES + 1):
try:
fd.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
if retry == RETRIES: raise
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
raise
print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry))
else:
break
for tc in test_cases: for tc in test_cases:
if not test_case.get('params', {}).get('skip_download', False): if not test_case.get('params', {}).get('skip_download', False):

View File

@ -308,5 +308,25 @@
"info_dict": { "info_dict": {
"title": "Vulkanausbruch in Ecuador: Der \"Feuerschlund\" ist wieder aktiv" "title": "Vulkanausbruch in Ecuador: Der \"Feuerschlund\" ist wieder aktiv"
} }
},
{
"name": "LiveLeak",
"md5": "0813c2430bea7a46bf13acf3406992f4",
"url": "http://www.liveleak.com/view?i=757_1364311680",
"file": "757_1364311680.mp4",
"info_dict": {
"title": "Most unlucky car accident",
"description": "extremely bad day for this guy..!",
"uploader": "ljfriel2"
}
},
{
"name": "WorldStarHipHop",
"url": "http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO",
"file": "wshh6a7q1ny0G34ZwuIO.mp4",
"md5": "9d04de741161603bf7071bbf4e883186",
"info_dict": {
"title": "Video: KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick! "
}
} }
] ]

Binary file not shown.

View File

@ -231,11 +231,21 @@ def trouble(self, message=None, tb=None):
self.to_stderr(message) self.to_stderr(message)
if self.params.get('verbose'): if self.params.get('verbose'):
if tb is None: if tb is None:
tb_data = traceback.format_list(traceback.extract_stack()) if sys.exc_info()[0]: # if .trouble has been called from an except block
tb = u''.join(tb_data) tb = u''
if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
tb += compat_str(traceback.format_exc())
else:
tb_data = traceback.format_list(traceback.extract_stack())
tb = u''.join(tb_data)
self.to_stderr(tb) self.to_stderr(tb)
if not self.params.get('ignoreerrors', False): if not self.params.get('ignoreerrors', False):
raise DownloadError(message) if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
exc_info = sys.exc_info()[1].exc_info
else:
exc_info = sys.exc_info()
raise DownloadError(message, exc_info)
self._download_retcode = 1 self._download_retcode = 1
def report_warning(self, message): def report_warning(self, message):
@ -250,6 +260,18 @@ def report_warning(self, message):
warning_message=u'%s %s' % (_msg_header,message) warning_message=u'%s %s' % (_msg_header,message)
self.to_stderr(warning_message) self.to_stderr(warning_message)
def report_error(self, message, tb=None):
'''
Do the same as trouble, but prefixes the message with 'ERROR:', colored
in red if stderr is a tty file.
'''
if sys.stderr.isatty():
_msg_header = u'\033[0;31mERROR:\033[0m'
else:
_msg_header = u'ERROR:'
error_message = u'%s %s' % (_msg_header, message)
self.trouble(error_message, tb)
def slow_down(self, start_time, byte_counter): def slow_down(self, start_time, byte_counter):
"""Sleep if the download speed is over the rate limit.""" """Sleep if the download speed is over the rate limit."""
rate_limit = self.params.get('ratelimit', None) rate_limit = self.params.get('ratelimit', None)
@ -281,7 +303,7 @@ def try_rename(self, old_filename, new_filename):
return return
os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
except (IOError, OSError) as err: except (IOError, OSError) as err:
self.trouble(u'ERROR: unable to rename file') self.report_error(u'unable to rename file')
def try_utime(self, filename, last_modified_hdr): def try_utime(self, filename, last_modified_hdr):
"""Try to set the last-modified time of the given file.""" """Try to set the last-modified time of the given file."""
@ -519,7 +541,7 @@ def process_info(self, info_dict):
if dn != '' and not os.path.exists(dn): # dn is already encoded if dn != '' and not os.path.exists(dn): # dn is already encoded
os.makedirs(dn) os.makedirs(dn)
except (OSError, IOError) as err: except (OSError, IOError) as err:
self.trouble(u'ERROR: unable to create directory ' + compat_str(err)) self.report_error(u'unable to create directory ' + compat_str(err))
return return
if self.params.get('writedescription', False): if self.params.get('writedescription', False):
@ -529,7 +551,7 @@ def process_info(self, info_dict):
with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile: with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
descfile.write(info_dict['description']) descfile.write(info_dict['description'])
except (OSError, IOError): except (OSError, IOError):
self.trouble(u'ERROR: Cannot write description file ' + descfn) self.report_error(u'Cannot write description file ' + descfn)
return return
if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']: if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
@ -538,14 +560,17 @@ def process_info(self, info_dict):
subtitle = info_dict['subtitles'][0] subtitle = info_dict['subtitles'][0]
(sub_error, sub_lang, sub) = subtitle (sub_error, sub_lang, sub) = subtitle
sub_format = self.params.get('subtitlesformat') sub_format = self.params.get('subtitlesformat')
try: if sub_error:
sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format self.report_warning("Some error while getting the subtitles")
self.report_writesubtitles(sub_filename) else:
with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: try:
subfile.write(sub) sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
except (OSError, IOError): self.report_writesubtitles(sub_filename)
self.trouble(u'ERROR: Cannot write subtitles file ' + descfn) with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
return subfile.write(sub)
except (OSError, IOError):
self.report_error(u'Cannot write subtitles file ' + descfn)
return
if self.params.get('onlysubtitles', False): if self.params.get('onlysubtitles', False):
return return
@ -554,14 +579,17 @@ def process_info(self, info_dict):
sub_format = self.params.get('subtitlesformat') sub_format = self.params.get('subtitlesformat')
for subtitle in subtitles: for subtitle in subtitles:
(sub_error, sub_lang, sub) = subtitle (sub_error, sub_lang, sub) = subtitle
try: if sub_error:
sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format self.report_warning("Some error while getting the subtitles")
self.report_writesubtitles(sub_filename) else:
with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: try:
subfile.write(sub) sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
except (OSError, IOError): self.report_writesubtitles(sub_filename)
self.trouble(u'ERROR: Cannot write subtitles file ' + descfn) with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
return subfile.write(sub)
except (OSError, IOError):
self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
return
if self.params.get('onlysubtitles', False): if self.params.get('onlysubtitles', False):
return return
@ -572,7 +600,7 @@ def process_info(self, info_dict):
json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle']) json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
write_json_file(json_info_dict, encodeFilename(infofn)) write_json_file(json_info_dict, encodeFilename(infofn))
except (OSError, IOError): except (OSError, IOError):
self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn) self.report_error(u'Cannot write metadata to JSON file ' + infofn)
return return
if not self.params.get('skip_download', False): if not self.params.get('skip_download', False):
@ -584,17 +612,17 @@ def process_info(self, info_dict):
except (OSError, IOError) as err: except (OSError, IOError) as err:
raise UnavailableVideoError() raise UnavailableVideoError()
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self.trouble(u'ERROR: unable to download video data: %s' % str(err)) self.report_error(u'unable to download video data: %s' % str(err))
return return
except (ContentTooShortError, ) as err: except (ContentTooShortError, ) as err:
self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
return return
if success: if success:
try: try:
self.post_process(filename, info_dict) self.post_process(filename, info_dict)
except (PostProcessingError) as err: except (PostProcessingError) as err:
self.trouble(u'ERROR: postprocessing: %s' % str(err)) self.report_error(u'postprocessing: %s' % str(err))
return return
def download(self, url_list): def download(self, url_list):
@ -611,6 +639,9 @@ def download(self, url_list):
self.process_info(video) self.process_info(video)
except UnavailableVideoError: except UnavailableVideoError:
self.trouble(u'\nERROR: unable to download video') self.trouble(u'\nERROR: unable to download video')
except MaxDownloadsReached:
self.to_screen(u'[info] Maximum number of downloaded files reached.')
raise
return self._download_retcode return self._download_retcode
@ -645,7 +676,7 @@ def _download_with_rtmpdump(self, filename, url, player_url, page_url):
try: try:
subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
except (OSError, IOError): except (OSError, IOError):
self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run') self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
return False return False
# Download using rtmpdump. rtmpdump returns exit code 2 when # Download using rtmpdump. rtmpdump returns exit code 2 when
@ -690,7 +721,8 @@ def _download_with_rtmpdump(self, filename, url, player_url, page_url):
}) })
return True return True
else: else:
self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval) self.to_stderr(u"\n")
self.report_error(u'rtmpdump exited with code %d' % retval)
return False return False
def _do_download(self, filename, info_dict): def _do_download(self, filename, info_dict):
@ -790,7 +822,7 @@ def _do_download(self, filename, info_dict):
self.report_retry(count, retries) self.report_retry(count, retries)
if count > retries: if count > retries:
self.trouble(u'ERROR: giving up after %s retries' % retries) self.report_error(u'giving up after %s retries' % retries)
return False return False
data_len = data.info().get('Content-length', None) data_len = data.info().get('Content-length', None)
@ -826,12 +858,13 @@ def _do_download(self, filename, info_dict):
filename = self.undo_temp_name(tmpfilename) filename = self.undo_temp_name(tmpfilename)
self.report_destination(filename) self.report_destination(filename)
except (OSError, IOError) as err: except (OSError, IOError) as err:
self.trouble(u'ERROR: unable to open for writing: %s' % str(err)) self.report_error(u'unable to open for writing: %s' % str(err))
return False return False
try: try:
stream.write(data_block) stream.write(data_block)
except (IOError, OSError) as err: except (IOError, OSError) as err:
self.trouble(u'\nERROR: unable to write data: %s' % str(err)) self.to_stderr(u"\n")
self.report_error(u'unable to write data: %s' % str(err))
return False return False
if not self.params.get('noresizebuffer', False): if not self.params.get('noresizebuffer', False):
block_size = self.best_block_size(after - before, len(data_block)) block_size = self.best_block_size(after - before, len(data_block))
@ -857,7 +890,8 @@ def _do_download(self, filename, info_dict):
self.slow_down(start, byte_counter - resume_len) self.slow_down(start, byte_counter - resume_len)
if stream is None: if stream is None:
self.trouble(u'\nERROR: Did not get any data blocks') self.to_stderr(u"\n")
self.report_error(u'Did not get any data blocks')
return False return False
stream.close() stream.close()
self.report_finish() self.report_finish()

File diff suppressed because it is too large Load Diff

View File

@ -311,7 +311,7 @@ def clean_html(html):
html = re.sub('<.*?>', '', html) html = re.sub('<.*?>', '', html)
# Replace html entities # Replace html entities
html = unescapeHTML(html) html = unescapeHTML(html)
return html return html.strip()
def sanitize_open(filename, open_mode): def sanitize_open(filename, open_mode):
@ -329,7 +329,7 @@ def sanitize_open(filename, open_mode):
if sys.platform == 'win32': if sys.platform == 'win32':
import msvcrt import msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
return (sys.stdout, filename) return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
stream = open(encodeFilename(filename), open_mode) stream = open(encodeFilename(filename), open_mode)
return (stream, filename) return (stream, filename)
except (IOError, OSError) as err: except (IOError, OSError) as err:
@ -435,6 +435,7 @@ def __init__(self, msg, tb=None):
""" tb, if given, is the original traceback (so that it can be printed out). """ """ tb, if given, is the original traceback (so that it can be printed out). """
super(ExtractorError, self).__init__(msg) super(ExtractorError, self).__init__(msg)
self.traceback = tb self.traceback = tb
self.exc_info = sys.exc_info() # preserve original exception
def format_traceback(self): def format_traceback(self):
if self.traceback is None: if self.traceback is None:
@ -449,7 +450,10 @@ class DownloadError(Exception):
configured to continue on errors. They will contain the appropriate configured to continue on errors. They will contain the appropriate
error message. error message.
""" """
pass def __init__(self, msg, exc_info=None):
""" exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
super(DownloadError, self).__init__(msg)
self.exc_info = exc_info
class SameFileError(Exception): class SameFileError(Exception):

View File

@ -1,2 +1,2 @@
__version__ = '2013.02.25' __version__ = '2013.04.03'