diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index 704ae6f5a..3bc41e5b2 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -58,6 +58,7 @@ class DownloadContext(dict):
         ctx.block_size = self.params.get('buffersize', 1024)
         ctx.start_time = time.time()
         ctx.chunk_size = None
+        throttle_start = None
 
         if self.params.get('continuedl', True):
             # Establish possible resume length
@@ -197,6 +198,7 @@ def establish_connection():
                 raise RetryDownload(err)
 
         def download():
+            nonlocal throttle_start
             data_len = ctx.data.info().get('Content-length', None)
 
             # Range HTTP header may be ignored/unsupported by a webserver
@@ -225,7 +227,6 @@ def download():
             # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
             now = None  # needed for slow_down() in the first loop run
             before = start  # start measuring
-            throttle_start = None
 
             def retry(e):
                 to_stdout = ctx.tmpfilename == '-'
@@ -326,7 +327,7 @@ def retry(e):
                         if ctx.stream is not None and ctx.tmpfilename != '-':
                             ctx.stream.close()
                         raise ThrottledDownload()
-                else:
+                elif speed:
                     throttle_start = None
 
             if not is_test and ctx.chunk_size and ctx.data_len is not None and byte_counter < ctx.data_len: