yt-dlp/youtube_dlc/downloader/youtube_live_chat.py

98 lines
3.7 KiB
Python
Raw Normal View History

2020-08-04 22:02:23 +00:00
from __future__ import division, unicode_literals
import re
import json
from .fragment import FragmentFD
class YoutubeLiveChatReplayFD(FragmentFD):
""" Downloads YouTube live chat replays fragment by fragment """
FD_NAME = 'youtube_live_chat_replay'
def real_download(self, filename, info_dict):
video_id = info_dict['video_id']
self.to_screen('[%s] Downloading live chat' % self.FD_NAME)
test = self.params.get('test', False)
ctx = {
'filename': filename,
'live': True,
'total_frags': None,
}
def dl_fragment(url):
headers = info_dict.get('http_headers', {})
return self._download_fragment(ctx, url, info_dict, headers)
def parse_yt_initial_data(data):
2020-08-10 21:13:43 +00:00
window_patt = b'window\\["ytInitialData"\\]\\s*=\\s*(.*?)(?<=});'
var_patt = b'var\\s+ytInitialData\\s*=\\s*(.*?)(?<=});'
2020-08-10 21:05:32 +00:00
for patt in window_patt, var_patt:
try:
raw_json = re.search(patt, data).group(1)
return json.loads(raw_json)
except AttributeError:
continue
2020-08-04 22:02:23 +00:00
self._prepare_and_start_frag_download(ctx)
success, raw_fragment = dl_fragment(
'https://www.youtube.com/watch?v={}'.format(video_id))
if not success:
return False
data = parse_yt_initial_data(raw_fragment)
continuation_id = data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
# no data yet but required to call _append_fragment
self._append_fragment(ctx, b'')
first = True
offset = None
while continuation_id is not None:
data = None
if first:
url = 'https://www.youtube.com/live_chat_replay?continuation={}'.format(continuation_id)
success, raw_fragment = dl_fragment(url)
if not success:
return False
data = parse_yt_initial_data(raw_fragment)
else:
url = ('https://www.youtube.com/live_chat_replay/get_live_chat_replay'
+ '?continuation={}'.format(continuation_id)
2020-11-28 00:19:38 +00:00
+ '&playerOffsetMs={}'.format(max(offset - 5000, 0))
2020-08-04 22:02:23 +00:00
+ '&hidden=false'
+ '&pbj=1')
success, raw_fragment = dl_fragment(url)
if not success:
return False
data = json.loads(raw_fragment)['response']
first = False
continuation_id = None
live_chat_continuation = data['continuationContents']['liveChatContinuation']
offset = None
processed_fragment = bytearray()
if 'actions' in live_chat_continuation:
for action in live_chat_continuation['actions']:
if 'replayChatItemAction' in action:
replay_chat_item_action = action['replayChatItemAction']
offset = int(replay_chat_item_action['videoOffsetTimeMsec'])
processed_fragment.extend(
json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n')
[youtube] Fix 'liveChatReplayContinuationData' missing 'continuation' key live_chat_continuation['continuations'][0]['liveChatReplayContinuationData']['continuation'] can not exist. So catch the KeyError. Traceback: $ tubeup 'https://youtube.com/watch?v=JyE9OF03cao' [debug] Encodings: locale UTF-8, fs utf-8, out UTF-8, pref UTF-8 [debug] youtube-dlc version 2020.10.25 [debug] Python version 3.7.3 (CPython) - Linux-5.8.0-0.bpo.2-amd64-x86_64-with-debian-10.6 [debug] exe versions: ffmpeg 3.3.9, ffprobe 3.3.9 [debug] Proxy map: {} There are no annotations to write. [download] 452.59KiB at 615.35KiB/s (00:01)ERROR: 'liveChatReplayContinuationData' Traceback (most recent call last): File "/mnt/data2/Backup/Wiki/.local/lib/python3.7/site-packages/youtube_dlc/YoutubeDL.py", line 846, in extract_info return self.process_ie_result(ie_result, download, extra_info) File "/mnt/data2/Backup/Wiki/.local/lib/python3.7/site-packages/youtube_dlc/YoutubeDL.py", line 901, in process_ie_result return self.process_video_result(ie_result, download=download) File "/mnt/data2/Backup/Wiki/.local/lib/python3.7/site-packages/youtube_dlc/YoutubeDL.py", line 1696, in process_video_result self.process_info(new_info) File "/mnt/data2/Backup/Wiki/.local/lib/python3.7/site-packages/youtube_dlc/YoutubeDL.py", line 1894, in process_info dl(sub_filename, sub_info, subtitle=True) File "/mnt/data2/Backup/Wiki/.local/lib/python3.7/site-packages/youtube_dlc/YoutubeDL.py", line 1866, in dl return fd.download(name, info, subtitle) File "/mnt/data2/Backup/Wiki/.local/lib/python3.7/site-packages/youtube_dlc/downloader/common.py", line 375, in download return self.real_download(filename, info_dict) File "/mnt/data2/Backup/Wiki/.local/lib/python3.7/site-packages/youtube_dlc/downloader/youtube_live_chat.py", line 85, in real_download continuation_id = live_chat_continuation['continuations'][0]['liveChatReplayContinuationData']['continuation'] KeyError: 'liveChatReplayContinuationData'
2020-11-08 07:49:03 +00:00
try:
continuation_id = live_chat_continuation['continuations'][0]['liveChatReplayContinuationData']['continuation']
except KeyError:
continuation_id = None
2020-08-04 22:02:23 +00:00
self._append_fragment(ctx, processed_fragment)
if test or offset is None:
break
self._finish_frag_download(ctx)
return True