From f3bdae76de7265475277c265f9fff12cdc59825b Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 3 Mar 2016 10:27:22 +0100 Subject: [PATCH 1/5] [extractor/common] add update_url_params helper method to add or update query string params --- youtube_dl/extractor/common.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 402f2f436..282559597 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -517,6 +517,13 @@ def _parse_json(self, json_string, video_id, transform_source=None, fatal=True): else: self.report_warning(errmsg + str(ve)) + def update_url_params(self, url, params): + parsed_url = compat_urlparse.urlparse(url) + qs = compat_urlparse.parse_qs(parsed_url.query) + qs.update(params) + return compat_urlparse.urlunparse( + parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True))) + def report_warning(self, msg, video_id=None): idstr = '' if video_id is None else '%s: ' % video_id self._downloader.report_warning( From fa9e259fd91ce1bf31310330adc20ddef2b1d948 Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 3 Mar 2016 10:54:39 +0100 Subject: [PATCH 2/5] [extractor/common] use compat_parse_qs in update_url_params --- youtube_dl/extractor/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 282559597..a95387cee 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -20,6 +20,7 @@ compat_urllib_error, compat_urllib_parse, compat_urlparse, + compat_parse_qs, compat_str, compat_etree_fromstring, ) @@ -519,7 +520,7 @@ def _parse_json(self, json_string, video_id, transform_source=None, fatal=True): def update_url_params(self, url, params): parsed_url = compat_urlparse.urlparse(url) - qs = compat_urlparse.parse_qs(parsed_url.query) + qs = compat_parse_qs(parsed_url.query) qs.update(params) return compat_urlparse.urlunparse( parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True))) From 38f9ef31dc434a6702686844b421085955137c55 Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 3 Mar 2016 18:34:52 +0100 Subject: [PATCH 3/5] [utils] add update_url_query function --- youtube_dl/extractor/common.py | 8 -------- youtube_dl/utils.py | 8 ++++++++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index a95387cee..402f2f436 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -20,7 +20,6 @@ compat_urllib_error, compat_urllib_parse, compat_urlparse, - compat_parse_qs, compat_str, compat_etree_fromstring, ) @@ -518,13 +517,6 @@ def _parse_json(self, json_string, video_id, transform_source=None, fatal=True): else: self.report_warning(errmsg + str(ve)) - def update_url_params(self, url, params): - parsed_url = compat_urlparse.urlparse(url) - qs = compat_parse_qs(parsed_url.query) - qs.update(params) - return compat_urlparse.urlunparse( - parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True))) - def report_warning(self, msg, video_id=None): idstr = '' if video_id is None else '%s: ' % video_id self._downloader.report_warning( diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 210c47fce..31d60f323 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1719,6 +1719,14 @@ def urlencode_postdata(*args, **kargs): return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii') +def update_url_query(url, query): + parsed_url = compat_urlparse.urlparse(url) + qs = compat_parse_qs(parsed_url.query) + qs.update(query) + return compat_urlparse.urlunparse(parsed_url._replace( + query=compat_urllib_parse.urlencode(qs, True))) + + def encode_dict(d, encoding='utf-8'): def encode(v): return v.encode(encoding) if isinstance(v, compat_basestring) else v From fb640d0a3d3af77e6835d9a6a2bf80920785511a Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 3 Mar 2016 18:40:05 +0100 Subject: [PATCH 4/5] [test/test_utils] add tests for update_url_query --- test/test_utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/test_utils.py b/test/test_utils.py index 97587ad2f..19a546619 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -61,6 +61,7 @@ lowercase_escape, url_basename, urlencode_postdata, + update_url_query, version_tuple, xpath_with_ns, xpath_element, @@ -76,6 +77,8 @@ ) from youtube_dl.compat import ( compat_etree_fromstring, + compat_urlparse, + compat_parse_qs, ) @@ -454,6 +457,31 @@ def test_urlencode_postdata(self): data = urlencode_postdata({'username': 'foo@bar.com', 'password': '1234'}) self.assertTrue(isinstance(data, bytes)) + def test_update_url_query(self): + def query_dict(url): + return compat_parse_qs(compat_urlparse.urlparse(url).query) + self.assertEqual(query_dict(update_url_query( + 'http://example.com/path', {'quality': ['HD'], 'format': ['mp4']})), + query_dict('http://example.com/path?quality=HD&format=mp4')) + self.assertEqual(query_dict(update_url_query( + 'http://example.com/path', {'system': ['LINUX', 'WINDOWS']})), + query_dict('http://example.com/path?system=LINUX&system=WINDOWS')) + self.assertEqual(query_dict(update_url_query( + 'http://example.com/path', {'fields': 'id,formats,subtitles'})), + query_dict('http://example.com/path?fields=id,formats,subtitles')) + self.assertEqual(query_dict(update_url_query( + 'http://example.com/path', {'fields': ('id,formats,subtitles', 'thumbnails')})), + query_dict('http://example.com/path?fields=id,formats,subtitles&fields=thumbnails')) + self.assertEqual(query_dict(update_url_query( + 'http://example.com/path?manifest=f4m', {'manifest': []})), + query_dict('http://example.com/path')) + self.assertEqual(query_dict(update_url_query( + 'http://example.com/path?system=LINUX&system=WINDOWS', {'system': 'LINUX'})), + query_dict('http://example.com/path?system=LINUX')) + self.assertEqual(query_dict(update_url_query( + 'http://example.com/path', {'fields': b'id,formats,subtitles'})), + query_dict('http://example.com/path?fields=id,formats,subtitles')) + def test_dict_get(self): FALSE_VALUES = { 'none': None, From 3201a67f618f3bb7b13b4df2a9c7f68f481940a4 Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 3 Mar 2016 19:18:57 +0100 Subject: [PATCH 5/5] [test/test_utils] add more tests for update_url_query --- test/test_utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_utils.py b/test/test_utils.py index 19a546619..2bcf8ecf0 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -481,6 +481,15 @@ def query_dict(url): self.assertEqual(query_dict(update_url_query( 'http://example.com/path', {'fields': b'id,formats,subtitles'})), query_dict('http://example.com/path?fields=id,formats,subtitles')) + self.assertEqual(query_dict(update_url_query( + 'http://example.com/path', {'width': 1080, 'height': 720})), + query_dict('http://example.com/path?width=1080&height=720')) + self.assertEqual(query_dict(update_url_query( + 'http://example.com/path', {'bitrate': 5020.43})), + query_dict('http://example.com/path?bitrate=5020.43')) + self.assertEqual(query_dict(update_url_query( + 'http://example.com/path', {'test': '第二行тест'})), + query_dict('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82')) def test_dict_get(self): FALSE_VALUES = {