[XAttrMetadata] Refactor and document dependencies

This commit is contained in:
pukkandan 2022-05-01 04:46:05 +05:30
parent 43d7f5a5d0
commit 6f7563beb7
No known key found for this signature in database
GPG Key ID: 7EEE9E1E817D0A39
5 changed files with 81 additions and 118 deletions

View File

@ -273,6 +273,7 @@ ## DEPENDENCIES
* [**secretstorage**](https://github.com/mitya57/secretstorage) - For accessing the Gnome keyring while decrypting cookies of Chromium-based browsers on Linux. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
* [**brotli**](https://github.com/google/brotli)\* or [**brotlicffi**](https://github.com/python-hyper/brotlicffi) - [Brotli](https://en.wikipedia.org/wiki/Brotli) content encoding support. Both licensed under MIT <sup>[1](https://github.com/google/brotli/blob/master/LICENSE) [2](https://github.com/python-hyper/brotlicffi/blob/master/LICENSE) </sup>
* [**certifi**](https://github.com/certifi/python-certifi)\* - Provides Mozilla's root certificate bundle. Licensed under [MPLv2](https://github.com/certifi/python-certifi/blob/master/LICENSE)
* [**xattr**](https://github.com/xattr/xattr), [**pyxattr**](https://github.com/iustin/pyxattr) or [**setfattr**](http://savannah.nongnu.org/projects/attr) - For writing xattr metadata on Linux. Licensed under [MIT](https://github.com/xattr/xattr/blob/master/LICENSE.txt), [LGPL2.1](https://github.com/iustin/pyxattr/blob/master/COPYING) and [GPLv2+](http://git.savannah.nongnu.org/cgit/attr.git/tree/doc/COPYING) respectively
* [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen/ffmpeg cannot. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
* [**rtmpdump**](http://rtmpdump.mplayerhq.hu) - For downloading `rtmp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](http://rtmpdump.mplayerhq.hu)
* [**mplayer**](http://mplayerhq.hu/design7/info.html) or [**mpv**](https://mpv.io) - For downloading `rstp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](https://github.com/mpv-player/mpv/blob/master/Copyright)

View File

@ -75,6 +75,15 @@
websockets = None
try:
import xattr # xattr or pyxattr
except ImportError:
xattr = None
else:
if hasattr(xattr, 'set'): # pyxattr
xattr._yt_dlp__identifier = 'pyxattr'
all_dependencies = {k: v for k, v in globals().items() if not k.startswith('_')}

View File

@ -1422,7 +1422,7 @@ def _dict_from_options_callback(
dest='parse_metadata', metavar='FIELDS REGEX REPLACE', action='append', nargs=3,
help='Replace text in a metadata field using the given regex. This option can be used multiple times')
postproc.add_option(
'--xattrs',
'--xattrs', '--xattr',
action='store_true', dest='xattrs', default=False,
help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
postproc.add_option(

View File

@ -12,68 +12,52 @@
class XAttrMetadataPP(PostProcessor):
#
# More info about extended attributes for media:
# http://freedesktop.org/wiki/CommonExtendedAttributes/
# http://www.freedesktop.org/wiki/PhreedomDraft/
# http://dublincore.org/documents/usageguide/elements.shtml
#
# TODO:
# * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
# * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
#
"""Set extended attributes on downloaded file (if xattr support is found)
More info about extended attributes for media:
http://freedesktop.org/wiki/CommonExtendedAttributes/
http://www.freedesktop.org/wiki/PhreedomDraft/
http://dublincore.org/documents/usageguide/elements.shtml
TODO:
* capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
* figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
"""
XATTR_MAPPING = {
'user.xdg.referrer.url': 'webpage_url',
# 'user.xdg.comment': 'description',
'user.dublincore.title': 'title',
'user.dublincore.date': 'upload_date',
'user.dublincore.description': 'description',
'user.dublincore.contributor': 'uploader',
'user.dublincore.format': 'format',
}
def run(self, info):
""" Set extended attributes on downloaded file (if xattr support is found). """
# Write the metadata to the file's xattrs
mtime = os.stat(info['filepath']).st_mtime
self.to_screen('Writing metadata to file\'s xattrs')
filename = info['filepath']
mtime = os.stat(filename).st_mtime
try:
xattr_mapping = {
'user.xdg.referrer.url': 'webpage_url',
# 'user.xdg.comment': 'description',
'user.dublincore.title': 'title',
'user.dublincore.date': 'upload_date',
'user.dublincore.description': 'description',
'user.dublincore.contributor': 'uploader',
'user.dublincore.format': 'format',
}
num_written = 0
for xattrname, infoname in xattr_mapping.items():
for xattrname, infoname in self.XATTR_MAPPING.items():
value = info.get(infoname)
if value:
if infoname == 'upload_date':
value = hyphenate_date(value)
byte_value = value.encode('utf-8')
write_xattr(filename, xattrname, byte_value)
num_written += 1
write_xattr(info['filepath'], xattrname, value.encode('utf-8'))
except XAttrUnavailableError as e:
raise PostProcessingError(str(e))
except XAttrMetadataError as e:
if e.reason == 'NO_SPACE':
self.report_warning(
'There\'s no disk space left, disk quota exceeded or filesystem xattr limit exceeded. '
+ (('Some ' if num_written else '') + 'extended attributes are not written.').capitalize())
'Some extended attributes are not written')
elif e.reason == 'VALUE_TOO_LONG':
self.report_warning(
'Unable to write extended attributes due to too long values.')
self.report_warning('Unable to write extended attributes due to too long values.')
else:
msg = 'This filesystem doesn\'t support extended attributes. '
if compat_os_name == 'nt':
msg += 'You need to use NTFS.'
else:
msg += '(You may have to enable them in your /etc/fstab)'
raise PostProcessingError(str(e))
tip = ('You need to use NTFS' if compat_os_name == 'nt'
else 'You may have to enable them in your "/etc/fstab"')
raise PostProcessingError(f'This filesystem doesn\'t support extended attributes. {tip}')
self.try_utime(filename, mtime, mtime)
self.try_utime(info['filepath'], mtime, mtime)
return [], info

View File

@ -4673,87 +4673,56 @@ def _get_pixel(idx):
def write_xattr(path, key, value):
# This mess below finds the best xattr tool for the job
try:
# try the pyxattr module...
import xattr
# Windows: Write xattrs to NTFS Alternate Data Streams:
# http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
if compat_os_name == 'nt':
assert ':' not in key
assert os.path.exists(path)
if hasattr(xattr, 'set'): # pyxattr
# Unicode arguments are not supported in python-pyxattr until
# version 0.5.0
# See https://github.com/ytdl-org/youtube-dl/issues/5498
pyxattr_required_version = '0.5.0'
if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
# TODO: fallback to CLI tools
raise XAttrUnavailableError(
'python-pyxattr is detected but is too old. '
'yt-dlp requires %s or above while your version is %s. '
'Falling back to other xattr implementations' % (
pyxattr_required_version, xattr.__version__))
try:
with open(f'{path}:{key}', 'wb') as f:
f.write(value)
except OSError as e:
raise XAttrMetadataError(e.errno, e.strerror)
return
# UNIX Method 1. Use xattrs/pyxattrs modules
from .dependencies import xattr
setxattr = None
if getattr(xattr, '_yt_dlp__identifier', None) == 'pyxattr':
# Unicode arguments are not supported in pyxattr until version 0.5.0
# See https://github.com/ytdl-org/youtube-dl/issues/5498
if version_tuple(xattr.__version__) >= (0, 5, 0):
setxattr = xattr.set
else: # xattr
setxattr = xattr.setxattr
elif xattr:
setxattr = xattr.setxattr
if setxattr:
try:
setxattr(path, key, value)
except OSError as e:
raise XAttrMetadataError(e.errno, e.strerror)
return
except ImportError:
if compat_os_name == 'nt':
# Write xattrs to NTFS Alternate Data Streams:
# http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
assert ':' not in key
assert os.path.exists(path)
# UNIX Method 2. Use setfattr/xattr executables
exe = ('setfattr' if check_executable('setfattr', ['--version'])
else 'xattr' if check_executable('xattr', ['-h']) else None)
if not exe:
raise XAttrUnavailableError(
'Couldn\'t find a tool to set the xattrs. Install either the python "xattr" or "pyxattr" modules or the '
+ ('"xattr" binary' if sys.platform != 'linux' else 'GNU "attr" package (which contains the "setfattr" tool)'))
ads_fn = path + ':' + key
try:
with open(ads_fn, 'wb') as f:
f.write(value)
except OSError as e:
raise XAttrMetadataError(e.errno, e.strerror)
else:
user_has_setfattr = check_executable('setfattr', ['--version'])
user_has_xattr = check_executable('xattr', ['-h'])
if user_has_setfattr or user_has_xattr:
value = value.decode('utf-8')
if user_has_setfattr:
executable = 'setfattr'
opts = ['-n', key, '-v', value]
elif user_has_xattr:
executable = 'xattr'
opts = ['-w', key, value]
cmd = ([encodeFilename(executable, True)]
+ [encodeArgument(o) for o in opts]
+ [encodeFilename(path, True)])
try:
p = Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
except OSError as e:
raise XAttrMetadataError(e.errno, e.strerror)
stdout, stderr = p.communicate_or_kill()
stderr = stderr.decode('utf-8', 'replace')
if p.returncode != 0:
raise XAttrMetadataError(p.returncode, stderr)
else:
# On Unix, and can't find pyxattr, setfattr, or xattr.
if sys.platform.startswith('linux'):
raise XAttrUnavailableError(
"Couldn't find a tool to set the xattrs. "
"Install either the python 'pyxattr' or 'xattr' "
"modules, or the GNU 'attr' package "
"(which contains the 'setfattr' tool).")
else:
raise XAttrUnavailableError(
"Couldn't find a tool to set the xattrs. "
"Install either the python 'xattr' module, "
"or the 'xattr' binary.")
value = value.decode('utf-8')
try:
p = Popen(
[exe, '-w', key, value, path] if exe == 'xattr' else [exe, '-n', key, '-v', value, path],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
except OSError as e:
raise XAttrMetadataError(e.errno, e.strerror)
stderr = p.communicate_or_kill()[1].decode('utf-8', 'replace')
if p.returncode:
raise XAttrMetadataError(p.returncode, stderr)
def random_birthday(year_field, month_field, day_field):