etc/yt-dlp
0
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-12-29 01:51:06 +00:00
Code Issues Projects Releases Wiki Activity

[utils] Fix InAdvancePagedList.__getitem__

Browse source 
Since it didn't have any cache, the page was re-fetched for each video.
* Also generalized the cache code
This commit is contained in:
pukkandan 2021-08-10 03:40:40 +05:30
parent 245d43cacf
commit 7be9ccff0b
1 changed files with 29 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
yt_dlp/utils.py
View file

@ -4041,15 +4041,31 @@ def __str__(self):
return repr(self.exhaust())
class PagedList(object):
class PagedList:
def __len__(self):
# This is only useful for tests
return len(self.getslice())
def getslice(self, start, end):
def __init__(self, pagefunc, pagesize, use_cache=True):
self._pagefunc = pagefunc
self._pagesize = pagesize
self._use_cache = use_cache
self._cache = {}
def getpage(self, pagenum):
page_results = self._cache.get(pagenum) or list(self._pagefunc(pagenum))
if self._use_cache:
self._cache[pagenum] = page_results
return page_results
def getslice(self, start=0, end=None):
return list(self._getslice(start, end))
def _getslice(self, start, end):
raise NotImplementedError('This method must be implemented by subclasses')
def __getitem__(self, idx):
# NOTE: cache must be enabled if this is used
if not isinstance(idx, int) or idx < 0:
raise TypeError('indices must be non-negative integers')
entries = self.getslice(idx, idx + 1)
@ -4057,42 +4073,26 @@ def __getitem__(self, idx):
class OnDemandPagedList(PagedList):
def __init__(self, pagefunc, pagesize, use_cache=True):
self._pagefunc = pagefunc
self._pagesize = pagesize
self._use_cache = use_cache
if use_cache:
self._cache = {}
def getslice(self, start=0, end=None):
res = []
def _getslice(self, start, end):
for pagenum in itertools.count(start // self._pagesize):
firstid = pagenum * self._pagesize
nextfirstid = pagenum * self._pagesize + self._pagesize
if start >= nextfirstid:
continue
page_results = None
if self._use_cache:
page_results = self._cache.get(pagenum)
if page_results is None:
page_results = list(self._pagefunc(pagenum))
if self._use_cache:
self._cache[pagenum] = page_results
startv = (
start % self._pagesize
if firstid <= start < nextfirstid
else 0)
endv = (
((end - 1) % self._pagesize) + 1
if (end is not None and firstid <= end <= nextfirstid)
else None)
page_results = self.getpage(pagenum)
if startv != 0 or endv is not None:
page_results = page_results[startv:endv]
res.extend(page_results)
yield from page_results
# A little optimization - if current page is not "full", ie. does
# not contain page_size videos then we can assume that this page
@ -4105,36 +4105,31 @@ def getslice(self, start=0, end=None):
# break out early as well
if end == nextfirstid:
break
return res
class InAdvancePagedList(PagedList):
def __init__(self, pagefunc, pagecount, pagesize):
self._pagefunc = pagefunc
self._pagecount = pagecount
self._pagesize = pagesize
PagedList.__init__(self, pagefunc, pagesize, True)
def getslice(self, start=0, end=None):
res = []
def _getslice(self, start, end):
start_page = start // self._pagesize
end_page = (
self._pagecount if end is None else (end // self._pagesize + 1))
skip_elems = start - start_page * self._pagesize
only_more = None if end is None else end - start
for pagenum in range(start_page, end_page):
page = list(self._pagefunc(pagenum))
page_results = self.getpage(pagenum)
if skip_elems:
page = page[skip_elems:]
page_results = page_results[skip_elems:]
skip_elems = None
if only_more is not None:
if len(page) < only_more:
only_more -= len(page)
if len(page_results) < only_more:
only_more -= len(page_results)
else:
page = page[:only_more]
res.extend(page)
yield from page_results[:only_more]
break
res.extend(page)
return res
yield from page_results
def uppercase_escape(s):