From 45806d44a7bc96dc0199410fec81cdce6015a546 Mon Sep 17 00:00:00 2001 From: Emanuel Hoogeveen <55082669+ehoogeveen-medweb@users.noreply.github.com> Date: Thu, 3 Mar 2022 15:33:32 +0100 Subject: [PATCH] [downloader] Obey `--file-access-retries` when deleting/renaming (#2224) Authored by: ehoogeveen-medweb --- yt_dlp/downloader/common.py | 49 ++++++++++++++++++++++------------- yt_dlp/downloader/external.py | 4 +-- yt_dlp/downloader/fragment.py | 4 +-- yt_dlp/options.py | 2 +- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 37321e34b..3a949d38a 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -210,28 +210,41 @@ def undo_temp_name(self, filename): def ytdl_filename(self, filename): return filename + '.ytdl' - def sanitize_open(self, filename, open_mode): - file_access_retries = self.params.get('file_access_retries', 10) - retry = 0 - while True: - try: - return sanitize_open(filename, open_mode) - except (IOError, OSError) as err: - retry = retry + 1 - if retry > file_access_retries or err.errno not in (errno.EACCES,): - raise - self.to_screen( - '[download] Got file access error. Retrying (attempt %d of %s) ...' - % (retry, self.format_retries(file_access_retries))) - time.sleep(0.01) + def wrap_file_access(action, *, fatal=False): + def outer(func): + def inner(self, *args, **kwargs): + file_access_retries = self.params.get('file_access_retries', 0) + retry = 0 + while True: + try: + return func(self, *args, **kwargs) + except (IOError, OSError) as err: + retry = retry + 1 + if retry > file_access_retries or err.errno not in (errno.EACCES, errno.EINVAL): + if not fatal: + self.report_error(f'unable to {action} file: {err}') + return + raise + self.to_screen( + f'[download] Unable to {action} file due to file access error. ' + f'Retrying (attempt {retry} of {self.format_retries(file_access_retries)}) ...') + time.sleep(0.01) + return inner + return outer + @wrap_file_access('open', fatal=True) + def sanitize_open(self, filename, open_mode): + return sanitize_open(filename, open_mode) + + @wrap_file_access('remove') + def try_remove(self, filename): + os.remove(filename) + + @wrap_file_access('rename') def try_rename(self, old_filename, new_filename): if old_filename == new_filename: return - try: - os.replace(old_filename, new_filename) - except (IOError, OSError) as err: - self.report_error(f'unable to rename file: {err}') + os.replace(old_filename, new_filename) def try_utime(self, filename, last_modified_hdr): """Try to set the last-modified time of the given file.""" diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 03ae3a00e..be6202eef 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -159,9 +159,9 @@ def _call_downloader(self, tmpfilename, info_dict): dest.write(decrypt_fragment(fragment, src.read())) src.close() if not self.params.get('keep_fragments', False): - os.remove(encodeFilename(fragment_filename)) + self.try_remove(encodeFilename(fragment_filename)) dest.close() - os.remove(encodeFilename('%s.frag.urls' % tmpfilename)) + self.try_remove(encodeFilename('%s.frag.urls' % tmpfilename)) return 0 diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py index 83a9f81b6..95fb2f9e7 100644 --- a/yt_dlp/downloader/fragment.py +++ b/yt_dlp/downloader/fragment.py @@ -159,7 +159,7 @@ def _append_fragment(self, ctx, frag_content): if self.__do_ytdl_file(ctx): self._write_ytdl_file(ctx) if not self.params.get('keep_fragments', False): - os.remove(encodeFilename(ctx['fragment_filename_sanitized'])) + self.try_remove(encodeFilename(ctx['fragment_filename_sanitized'])) del ctx['fragment_filename_sanitized'] def _prepare_frag_download(self, ctx): @@ -305,7 +305,7 @@ def _finish_frag_download(self, ctx, info_dict): if self.__do_ytdl_file(ctx): ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename'])) if os.path.isfile(ytdl_filename): - os.remove(ytdl_filename) + self.try_remove(ytdl_filename) elapsed = time.time() - ctx['started'] if ctx['tmpfilename'] == '-': diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 6fcef98cd..9908f3975 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -727,7 +727,7 @@ def _dict_from_options_callback( help='Number of retries (default is %default), or "infinite"') downloader.add_option( '--file-access-retries', - dest='file_access_retries', metavar='RETRIES', default=10, + dest='file_access_retries', metavar='RETRIES', default=3, help='Number of times to retry on file access error (default is %default), or "infinite"') downloader.add_option( '--fragment-retries',