ExtractorError for errors during extraction

youtube_dl/FileDownloader.py

@@ -210,7 +210,7 @@ def fixed_template(self):
         """Checks if the output template is fixed."""
         return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
 
-    def trouble(self, message=None):
+    def trouble(self, message=None, tb=None):
         """Determine action to take when a download problem appears.
 
         Depending on if the downloader has been configured to ignore
@@ -220,7 +220,9 @@ def trouble(self, message=None):
         if message is not None:
             self.to_stderr(message)
         if self.params.get('verbose'):
-            self.to_stderr(u''.join(traceback.format_list(traceback.extract_stack())))
+            if tb is None:
+                tb = u''.join(traceback.format_list(traceback.extract_stack()))
+            self.to_stderr(tb)
         if not self.params.get('ignoreerrors', False):
             raise DownloadError(message)
         self._download_retcode = 1
@@ -485,14 +487,24 @@ def download(self, url_list):
 
                 # Warn if the _WORKING attribute is False
                 if not ie.working():
-                    self.trouble(u'WARNING: the program functionality for this site has been marked as broken, '
+                    self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, '
-                                 u'and will probably not work. If you want to go on, use the -i option.')
+                                   u'and will probably not work. If you want to go on, use the -i option.')
 
                 # Suitable InfoExtractor found
                 suitable_found = True
 
                 # Extract information from URL and process it
-                videos = ie.extract(url)
+                try:
+                    videos = ie.extract(url)
+                except ExtractorError as de: # An error we somewhat expected
+                    self.trouble(u'ERROR: ' + compat_str(de), compat_str(u''.join(traceback.format_tb(de.traceback))))
+                    break
+                except Exception as e:
+                    if self.params.get('ignoreerrors', False):
+                        self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc()))
+                        break
+                    else:
+                        raise
 
                 if len(videos or []) > 1 and self.fixed_template():
                     raise SameFileError(self.params['outtmpl'])

youtube_dl/InfoExtractors.py

@@ -3808,8 +3808,7 @@ def _real_extract(self, url):
             webpage_bytes = urlh.read()
             webpage = webpage_bytes.decode('utf-8', 'ignore')
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+            raise ExtractorError(u'unable to download webpage: %s' % compat_str(err))
-            return
         m = re.search(r'data-title="(?P<title>.+)"',webpage)
         title = m.group('title')
         m = re.search(r'<a class="state" data-content-type="channel" data-content-id="(?P<uploader>\d+)"',webpage)

youtube_dl/utils.py

@@ -410,6 +410,17 @@ def encodeFilename(s):
     else:
         return s.encode(sys.getfilesystemencoding(), 'ignore')
 
+
+class ExtractorError(Exception):
+    """Error during info extraction."""
+    def __init__(self, msg, tb=None):
+        """ tb is the original traceback (so that it can be printed out) """
+        super(ExtractorError, self).__init__(msg)
+        if tb is None:
+            tb = sys.exc_info()[2]
+        self.traceback = tb
+
+
 class DownloadError(Exception):
     """Download Error exception.