From 0c14e2fbe3c5cba16af27525aa6715819226e6d8 Mon Sep 17 00:00:00 2001 From: pulpe Date: Sat, 22 Feb 2014 13:55:51 +0100 Subject: [PATCH 1/9] add post processor --- youtube_dl/__init__.py | 7 +++ youtube_dl/postprocessor/__init__.py | 4 ++ youtube_dl/postprocessor/atomicparsley.py | 59 +++++++++++++++++++++++ youtube_dl/postprocessor/ffmpeg.py | 14 ++++++ 4 files changed, 84 insertions(+) create mode 100644 youtube_dl/postprocessor/atomicparsley.py diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 6af4b8aee..490f76dc9 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -90,6 +90,8 @@ from .extractor import gen_extractors from .version import __version__ from .YoutubeDL import YoutubeDL from .postprocessor import ( + AtomicParsleyPP, + FFmpegMediaFixPP, FFmpegMetadataPP, FFmpegVideoConvertor, FFmpegExtractAudioPP, @@ -497,6 +499,8 @@ def parseOpts(overrideArguments=None): help='do not overwrite post-processed files; the post-processed files are overwritten by default') postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False, help='embed subtitles in the video (only for mp4 videos)') + postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False, + help='embed thumbnail in the audio as cover art') postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False, help='write metadata to the video file') postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False, @@ -803,6 +807,9 @@ def _real_main(argv=None): ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat)) if opts.xattrs: ydl.add_post_processor(XAttrMetadataPP()) + if opts.embedthumbnail: + ydl.add_post_processor(FFmpegMediaFixPP()) + ydl.add_post_processor(AtomicParsleyPP()) # Update version if opts.update_self: diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py index 7f19f717f..9410dcf5b 100644 --- a/youtube_dl/postprocessor/__init__.py +++ b/youtube_dl/postprocessor/__init__.py @@ -1,5 +1,7 @@ +from .atomicparsley import AtomicParsleyPP from .ffmpeg import ( + FFmpegMediaFixPP, FFmpegMergerPP, FFmpegMetadataPP, FFmpegVideoConvertor, @@ -9,6 +11,8 @@ from .ffmpeg import ( from .xattrpp import XAttrMetadataPP __all__ = [ + 'AtomicParsleyPP', + 'FFmpegMediaFixPP', 'FFmpegMergerPP', 'FFmpegMetadataPP', 'FFmpegVideoConvertor', diff --git a/youtube_dl/postprocessor/atomicparsley.py b/youtube_dl/postprocessor/atomicparsley.py new file mode 100644 index 000000000..0f7cf5417 --- /dev/null +++ b/youtube_dl/postprocessor/atomicparsley.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + + +import os +import subprocess + +from .common import PostProcessor + +from ..utils import ( + check_executable, + compat_urlretrieve, + encodeFilename, + PostProcessingError, + prepend_extension, + shell_quote +) + + +class AtomicParsleyPPError(PostProcessingError): + pass + + +class AtomicParsleyPP(PostProcessor): + def __init__(self, downloader=None): + PostProcessor.__init__(self, downloader) + + def run(self, info): + if not check_executable('AtomicParsley', ['-v']): + raise AtomicParsleyPPError('AtomicParsley was not found. Please install.') + + filename = info['filepath'] + temp_filename = prepend_extension(filename, 'temp') + temp_thumbnail = prepend_extension(filename, 'thumb') + + if not info.get('thumbnail'): + raise AtomicParsleyPPError('Thumbnail was not found. Nothing to do.') + + compat_urlretrieve(info['thumbnail'], temp_thumbnail) + + cmd = ['AtomicParsley', filename, '--artwork', temp_thumbnail, '-o', temp_filename] + + self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename) + + if self._downloader.params.get('verbose', False): + self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd)) + + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + + if p.returncode != 0: + msg = stderr.decode('utf-8', 'replace').strip() + raise AtomicParsleyPPError(msg) + + os.remove(encodeFilename(filename)) + os.remove(encodeFilename(temp_thumbnail)) + os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + + return True, info diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index c22f2cdc6..26a70491a 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -483,3 +483,17 @@ class FFmpegMergerPP(FFmpegPostProcessor): self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args) return True, info + +class FFmpegMediaFixPP(FFmpegPostProcessor): + def run(self, info): + filename = info['filepath'] + temp_filename = prepend_extension(filename, 'temp') + + options = ['-vcodec', 'copy', '-acodec', 'copy'] + self._downloader.to_screen(u'[ffmpeg] Fixing media file "%s"' % filename) + self.run_ffmpeg(filename, temp_filename, options) + + os.remove(encodeFilename(filename)) + os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + + return True, info From 149254d0d5fd9407a008e31c2e4362316e625d35 Mon Sep 17 00:00:00 2001 From: pulpe Date: Sat, 22 Feb 2014 18:10:07 +0100 Subject: [PATCH 2/9] fix ffmpeg error, if youtube-dl runs more than once with --embed-thumbnail with same video --- youtube_dl/__init__.py | 4 ++-- youtube_dl/postprocessor/__init__.py | 4 ++-- youtube_dl/postprocessor/ffmpeg.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 490f76dc9..8b95a6d9b 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -91,7 +91,7 @@ from .version import __version__ from .YoutubeDL import YoutubeDL from .postprocessor import ( AtomicParsleyPP, - FFmpegMediaFixPP, + FFmpegAudioFixPP, FFmpegMetadataPP, FFmpegVideoConvertor, FFmpegExtractAudioPP, @@ -808,7 +808,7 @@ def _real_main(argv=None): if opts.xattrs: ydl.add_post_processor(XAttrMetadataPP()) if opts.embedthumbnail: - ydl.add_post_processor(FFmpegMediaFixPP()) + ydl.add_post_processor(FFmpegAudioFixPP()) ydl.add_post_processor(AtomicParsleyPP()) # Update version diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py index 9410dcf5b..08e6ddd00 100644 --- a/youtube_dl/postprocessor/__init__.py +++ b/youtube_dl/postprocessor/__init__.py @@ -1,7 +1,7 @@ from .atomicparsley import AtomicParsleyPP from .ffmpeg import ( - FFmpegMediaFixPP, + FFmpegAudioFixPP, FFmpegMergerPP, FFmpegMetadataPP, FFmpegVideoConvertor, @@ -12,7 +12,7 @@ from .xattrpp import XAttrMetadataPP __all__ = [ 'AtomicParsleyPP', - 'FFmpegMediaFixPP', + 'FFmpegAudioFixPP', 'FFmpegMergerPP', 'FFmpegMetadataPP', 'FFmpegVideoConvertor', diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 26a70491a..75ee87e1a 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -484,13 +484,13 @@ class FFmpegMergerPP(FFmpegPostProcessor): return True, info -class FFmpegMediaFixPP(FFmpegPostProcessor): +class FFmpegAudioFixPP(FFmpegPostProcessor): def run(self, info): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') - options = ['-vcodec', 'copy', '-acodec', 'copy'] - self._downloader.to_screen(u'[ffmpeg] Fixing media file "%s"' % filename) + options = ['-vn', '-acodec', 'copy'] + self._downloader.to_screen(u'[ffmpeg] Fixing audio file "%s"' % filename) self.run_ffmpeg(filename, temp_filename, options) os.remove(encodeFilename(filename)) From 39c68260c0b308e250119fa136c7e4f76d14dfea Mon Sep 17 00:00:00 2001 From: pulpe Date: Sat, 22 Feb 2014 18:23:30 +0100 Subject: [PATCH 3/9] fix ffmpeg metadatapp --- youtube_dl/postprocessor/ffmpeg.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 75ee87e1a..d5b71b96f 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -464,7 +464,11 @@ class FFmpegMetadataPP(FFmpegPostProcessor): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') - options = ['-c', 'copy'] + if info['ext'] == u'm4a': + options = ['-vn', '-acodec', 'copy'] + else: + options = ['-c', 'copy'] + for (name, value) in metadata.items(): options.extend(['-metadata', '%s=%s' % (name, value)]) From 784763c56500bfcdd6b539dffdffeae8dfb6b91e Mon Sep 17 00:00:00 2001 From: pulpe Date: Sat, 22 Feb 2014 18:31:54 +0100 Subject: [PATCH 4/9] we don't need to run ffmpeg more times --- youtube_dl/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 8b95a6d9b..4f4ec3871 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -808,7 +808,8 @@ def _real_main(argv=None): if opts.xattrs: ydl.add_post_processor(XAttrMetadataPP()) if opts.embedthumbnail: - ydl.add_post_processor(FFmpegAudioFixPP()) + if not opts.addmetadata: + ydl.add_post_processor(FFmpegAudioFixPP()) ydl.add_post_processor(AtomicParsleyPP()) # Update version From 6a4f3528c84eb5d4d1c527e83e2d9bfd7639d426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 11 Apr 2014 20:40:42 +0700 Subject: [PATCH 5/9] [firstpost] Fix extraction --- youtube_dl/extractor/firstpost.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/firstpost.py b/youtube_dl/extractor/firstpost.py index 7e3d1afd2..eccd8dde9 100644 --- a/youtube_dl/extractor/firstpost.py +++ b/youtube_dl/extractor/firstpost.py @@ -6,7 +6,6 @@ from .common import InfoExtractor class FirstpostIE(InfoExtractor): - IE_NAME = 'Firstpost.com' _VALID_URL = r'http://(?:www\.)?firstpost\.com/[^/]+/.*-(?P[0-9]+)\.html' _TEST = { @@ -16,7 +15,6 @@ class FirstpostIE(InfoExtractor): 'id': '1025403', 'ext': 'mp4', 'title': 'India to launch indigenous aircraft carrier INS Vikrant today', - 'description': 'Its flight deck is over twice the size of a football field, its power unit can light up the entire Kochi city and the cabling is enough to cover the distance between here to Delhi.', } } @@ -24,15 +22,26 @@ class FirstpostIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') - webpage = self._download_webpage(url, video_id) - video_url = self._html_search_regex( - r'', - webpage, 'video URL') + data = self._download_xml( + 'http://www.firstpost.com/getvideoxml-%s.xml' % video_id, video_id, + 'Downloading video XML') + + item = data.find('./playlist/item') + thumbnail = item.find('./image').text + title = item.find('./title').text + + formats = [ + { + 'url': details.find('./file').text, + 'format_id': details.find('./label').text.strip(), + 'width': int(details.find('./width').text.strip()), + 'height': int(details.find('./height').text.strip()), + } for details in item.findall('./source/file_details') if details.find('./file').text + ] return { 'id': video_id, - 'url': video_url, - 'title': self._og_search_title(webpage), - 'description': self._og_search_description(webpage), - 'thumbnail': self._og_search_thumbnail(webpage), + 'title': title, + 'thumbnail': thumbnail, + 'formats': formats, } From 97b5196960f46b0970fc12865998d8b98a51eb27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 11 Apr 2014 13:59:00 +0200 Subject: [PATCH 6/9] [weibo] Modernize --- youtube_dl/extractor/weibo.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py index fa784ab99..b24297a40 100644 --- a/youtube_dl/extractor/weibo.py +++ b/youtube_dl/extractor/weibo.py @@ -1,10 +1,11 @@ # coding: utf-8 +from __future__ import unicode_literals import re -import json from .common import InfoExtractor + class WeiboIE(InfoExtractor): """ The videos in Weibo come from different sites, this IE just finds the link @@ -13,16 +14,16 @@ class WeiboIE(InfoExtractor): _VALID_URL = r'https?://video\.weibo\.com/v/weishipin/t_(?P.+?)\.htm' _TEST = { - u'add_ie': ['Sina'], - u'url': u'http://video.weibo.com/v/weishipin/t_zjUw2kZ.htm', - u'file': u'98322879.flv', - u'info_dict': { - u'title': u'魔声耳机最新广告“All Eyes On Us”', + 'url': 'http://video.weibo.com/v/weishipin/t_zjUw2kZ.htm', + 'info_dict': { + 'id': '98322879', + 'ext': 'flv', + 'title': '魔声耳机最新广告“All Eyes On Us”', }, - u'note': u'Sina video', - u'params': { - u'skip_download': True, + 'params': { + 'skip_download': True, }, + 'add_ie': ['Sina'], } # Additional example videos from different sites @@ -33,17 +34,16 @@ class WeiboIE(InfoExtractor): mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE) video_id = mobj.group('id') info_url = 'http://video.weibo.com/?s=v&a=play_list&format=json&mix_video_id=t_%s' % video_id - info_page = self._download_webpage(info_url, video_id) - info = json.loads(info_page) + info = self._download_json(info_url, video_id) videos_urls = map(lambda v: v['play_page_url'], info['result']['data']) - #Prefer sina video since they have thumbnails - videos_urls = sorted(videos_urls, key=lambda u: u'video.sina.com' in u) + # Prefer sina video since they have thumbnails + videos_urls = sorted(videos_urls, key=lambda u: 'video.sina.com' in u) player_url = videos_urls[-1] - m_sina = re.match(r'https?://video.sina.com.cn/v/b/(\d+)-\d+.html', player_url) + m_sina = re.match(r'https?://video\.sina\.com\.cn/v/b/(\d+)-\d+\.html', + player_url) if m_sina is not None: self.to_screen('Sina video detected') sina_id = m_sina.group(1) player_url = 'http://you.video.sina.com.cn/swf/quotePlayer.swf?vid=%s' % sina_id return self.url_result(player_url) - From 37e341013745e89d38448f3740944fa4a299a145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 12 Apr 2014 02:53:55 +0700 Subject: [PATCH 7/9] [prosiebensat1] Add one more clip id pattern (Closes #2737) --- youtube_dl/extractor/prosiebensat1.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py index 3f585bebf..1e84b175f 100644 --- a/youtube_dl/extractor/prosiebensat1.py +++ b/youtube_dl/extractor/prosiebensat1.py @@ -160,6 +160,7 @@ class ProSiebenSat1IE(InfoExtractor): _CLIPID_REGEXES = [ r'"clip_id"\s*:\s+"(\d+)"', r'clipid: "(\d+)"', + r'clipId=(\d+)', ] _TITLE_REGEXES = [ r'

\s*(.+?)

', From 381640e3acf65e9eac52b6ad47d7f7cad45a7d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 12 Apr 2014 12:53:48 +0200 Subject: [PATCH 8/9] [brightcove] Only use url from meta element if it has the 'playerKey' field (fixes #2738) --- youtube_dl/extractor/brightcove.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index 339d60ff0..3c02c297a 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -140,7 +140,11 @@ class BrightcoveIE(InfoExtractor): url_m = re.search(r' Date: Sat, 12 Apr 2014 15:51:40 +0200 Subject: [PATCH 9/9] [atomicparsley] Remove unneeded __init__ method --- youtube_dl/postprocessor/atomicparsley.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/youtube_dl/postprocessor/atomicparsley.py b/youtube_dl/postprocessor/atomicparsley.py index 0f7cf5417..765b2d9ee 100644 --- a/youtube_dl/postprocessor/atomicparsley.py +++ b/youtube_dl/postprocessor/atomicparsley.py @@ -22,9 +22,6 @@ class AtomicParsleyPPError(PostProcessingError): class AtomicParsleyPP(PostProcessor): - def __init__(self, downloader=None): - PostProcessor.__init__(self, downloader) - def run(self, info): if not check_executable('AtomicParsley', ['-v']): raise AtomicParsleyPPError('AtomicParsley was not found. Please install.')