|
|
@ -259,14 +259,14 @@ def sanitize_open(filename, open_mode): |
|
|
|
import msvcrt |
|
|
|
import msvcrt |
|
|
|
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) |
|
|
|
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) |
|
|
|
return (sys.stdout, filename) |
|
|
|
return (sys.stdout, filename) |
|
|
|
stream = open(filename, open_mode) |
|
|
|
stream = open(_encodeFilename(filename), open_mode) |
|
|
|
return (stream, filename) |
|
|
|
return (stream, filename) |
|
|
|
except (IOError, OSError), err: |
|
|
|
except (IOError, OSError), err: |
|
|
|
# In case of error, try to remove win32 forbidden chars |
|
|
|
# In case of error, try to remove win32 forbidden chars |
|
|
|
filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename) |
|
|
|
filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename) |
|
|
|
|
|
|
|
|
|
|
|
# An exception here should be caught in the caller |
|
|
|
# An exception here should be caught in the caller |
|
|
|
stream = open(filename, open_mode) |
|
|
|
stream = open(_encodeFilename(filename), open_mode) |
|
|
|
return (stream, filename) |
|
|
|
return (stream, filename) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -299,6 +299,14 @@ def _unescapeHTML(s): |
|
|
|
htmlParser = HTMLParser.HTMLParser() |
|
|
|
htmlParser = HTMLParser.HTMLParser() |
|
|
|
return htmlParser.unescape(s) |
|
|
|
return htmlParser.unescape(s) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _encodeFilename(s): |
|
|
|
|
|
|
|
""" |
|
|
|
|
|
|
|
@param s The name of the file (of type unicode) |
|
|
|
|
|
|
|
""" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
assert type(s) == type(u'') |
|
|
|
|
|
|
|
return s.encode(sys.getfilesystemencoding(), 'ignore') |
|
|
|
|
|
|
|
|
|
|
|
class DownloadError(Exception): |
|
|
|
class DownloadError(Exception): |
|
|
|
"""Download Error exception. |
|
|
|
"""Download Error exception. |
|
|
|
|
|
|
|
|
|
|
@ -563,16 +571,17 @@ class FileDownloader(object): |
|
|
|
self._pps.append(pp) |
|
|
|
self._pps.append(pp) |
|
|
|
pp.set_downloader(self) |
|
|
|
pp.set_downloader(self) |
|
|
|
|
|
|
|
|
|
|
|
def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False): |
|
|
|
def to_screen(self, message, skip_eol=False): |
|
|
|
"""Print message to stdout if not in quiet mode.""" |
|
|
|
"""Print message to stdout if not in quiet mode.""" |
|
|
|
try: |
|
|
|
assert type(message) == type(u'') |
|
|
|
if not self.params.get('quiet', False): |
|
|
|
if not self.params.get('quiet', False): |
|
|
|
terminator = [u'\n', u''][skip_eol] |
|
|
|
terminator = [u'\n', u''][skip_eol] |
|
|
|
print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()), |
|
|
|
output = message + terminator |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr |
|
|
|
|
|
|
|
output = output.encode(preferredencoding(), 'ignore') |
|
|
|
|
|
|
|
self._screen_file.write(output) |
|
|
|
self._screen_file.flush() |
|
|
|
self._screen_file.flush() |
|
|
|
except (UnicodeEncodeError), err: |
|
|
|
|
|
|
|
if not ignore_encoding_errors: |
|
|
|
|
|
|
|
raise |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def to_stderr(self, message): |
|
|
|
def to_stderr(self, message): |
|
|
|
"""Print message to stderr.""" |
|
|
|
"""Print message to stderr.""" |
|
|
@ -622,7 +631,7 @@ class FileDownloader(object): |
|
|
|
def temp_name(self, filename): |
|
|
|
def temp_name(self, filename): |
|
|
|
"""Returns a temporary filename for the given filename.""" |
|
|
|
"""Returns a temporary filename for the given filename.""" |
|
|
|
if self.params.get('nopart', False) or filename == u'-' or \ |
|
|
|
if self.params.get('nopart', False) or filename == u'-' or \ |
|
|
|
(os.path.exists(filename) and not os.path.isfile(filename)): |
|
|
|
(os.path.exists(_encodeFilename(filename)) and not os.path.isfile(_encodeFilename(filename))): |
|
|
|
return filename |
|
|
|
return filename |
|
|
|
return filename + u'.part' |
|
|
|
return filename + u'.part' |
|
|
|
|
|
|
|
|
|
|
@ -635,7 +644,7 @@ class FileDownloader(object): |
|
|
|
try: |
|
|
|
try: |
|
|
|
if old_filename == new_filename: |
|
|
|
if old_filename == new_filename: |
|
|
|
return |
|
|
|
return |
|
|
|
os.rename(old_filename, new_filename) |
|
|
|
os.rename(_encodeFilename(old_filename), _encodeFilename(new_filename)) |
|
|
|
except (IOError, OSError), err: |
|
|
|
except (IOError, OSError), err: |
|
|
|
self.trouble(u'ERROR: unable to rename file') |
|
|
|
self.trouble(u'ERROR: unable to rename file') |
|
|
|
|
|
|
|
|
|
|
@ -643,7 +652,7 @@ class FileDownloader(object): |
|
|
|
"""Try to set the last-modified time of the given file.""" |
|
|
|
"""Try to set the last-modified time of the given file.""" |
|
|
|
if last_modified_hdr is None: |
|
|
|
if last_modified_hdr is None: |
|
|
|
return |
|
|
|
return |
|
|
|
if not os.path.isfile(filename): |
|
|
|
if not os.path.isfile(_encodeFilename(filename)): |
|
|
|
return |
|
|
|
return |
|
|
|
timestr = last_modified_hdr |
|
|
|
timestr = last_modified_hdr |
|
|
|
if timestr is None: |
|
|
|
if timestr is None: |
|
|
@ -659,15 +668,15 @@ class FileDownloader(object): |
|
|
|
|
|
|
|
|
|
|
|
def report_writedescription(self, descfn): |
|
|
|
def report_writedescription(self, descfn): |
|
|
|
""" Report that the description file is being written """ |
|
|
|
""" Report that the description file is being written """ |
|
|
|
self.to_screen(u'[info] Writing video description to: %s' % descfn, ignore_encoding_errors=True) |
|
|
|
self.to_screen(u'[info] Writing video description to: ' + descfn) |
|
|
|
|
|
|
|
|
|
|
|
def report_writeinfojson(self, infofn): |
|
|
|
def report_writeinfojson(self, infofn): |
|
|
|
""" Report that the metadata file has been written """ |
|
|
|
""" Report that the metadata file has been written """ |
|
|
|
self.to_screen(u'[info] Video description metadata as JSON to: %s' % infofn, ignore_encoding_errors=True) |
|
|
|
self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) |
|
|
|
|
|
|
|
|
|
|
|
def report_destination(self, filename): |
|
|
|
def report_destination(self, filename): |
|
|
|
"""Report destination filename.""" |
|
|
|
"""Report destination filename.""" |
|
|
|
self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True) |
|
|
|
self.to_screen(u'[download] Destination: ' + filename) |
|
|
|
|
|
|
|
|
|
|
|
def report_progress(self, percent_str, data_len_str, speed_str, eta_str): |
|
|
|
def report_progress(self, percent_str, data_len_str, speed_str, eta_str): |
|
|
|
"""Report download progress.""" |
|
|
|
"""Report download progress.""" |
|
|
@ -769,8 +778,8 @@ class FileDownloader(object): |
|
|
|
return |
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
try: |
|
|
|
try: |
|
|
|
dn = os.path.dirname(filename) |
|
|
|
dn = os.path.dirname(_encodeFilename(filename)) |
|
|
|
if dn != '' and not os.path.exists(dn): |
|
|
|
if dn != '' and not os.path.exists(dn): # dn is already encoded |
|
|
|
os.makedirs(dn) |
|
|
|
os.makedirs(dn) |
|
|
|
except (OSError, IOError), err: |
|
|
|
except (OSError, IOError), err: |
|
|
|
self.trouble(u'ERROR: unable to create directory ' + unicode(err)) |
|
|
|
self.trouble(u'ERROR: unable to create directory ' + unicode(err)) |
|
|
@ -778,9 +787,9 @@ class FileDownloader(object): |
|
|
|
|
|
|
|
|
|
|
|
if self.params.get('writedescription', False): |
|
|
|
if self.params.get('writedescription', False): |
|
|
|
try: |
|
|
|
try: |
|
|
|
descfn = filename + '.description' |
|
|
|
descfn = filename + u'.description' |
|
|
|
self.report_writedescription(descfn) |
|
|
|
self.report_writedescription(descfn) |
|
|
|
descfile = open(descfn, 'wb') |
|
|
|
descfile = open(_encodeFilename(descfn), 'wb') |
|
|
|
try: |
|
|
|
try: |
|
|
|
descfile.write(info_dict['description'].encode('utf-8')) |
|
|
|
descfile.write(info_dict['description'].encode('utf-8')) |
|
|
|
finally: |
|
|
|
finally: |
|
|
@ -790,7 +799,7 @@ class FileDownloader(object): |
|
|
|
return |
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
if self.params.get('writeinfojson', False): |
|
|
|
if self.params.get('writeinfojson', False): |
|
|
|
infofn = filename + '.info.json' |
|
|
|
infofn = filename + u'.info.json' |
|
|
|
self.report_writeinfojson(infofn) |
|
|
|
self.report_writeinfojson(infofn) |
|
|
|
try: |
|
|
|
try: |
|
|
|
json.dump |
|
|
|
json.dump |
|
|
@ -798,7 +807,7 @@ class FileDownloader(object): |
|
|
|
self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.') |
|
|
|
self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.') |
|
|
|
return |
|
|
|
return |
|
|
|
try: |
|
|
|
try: |
|
|
|
infof = open(infofn, 'wb') |
|
|
|
infof = open(_encodeFilename(infofn), 'wb') |
|
|
|
try: |
|
|
|
try: |
|
|
|
json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',)) |
|
|
|
json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',)) |
|
|
|
json.dump(json_info_dict, infof) |
|
|
|
json.dump(json_info_dict, infof) |
|
|
@ -809,7 +818,7 @@ class FileDownloader(object): |
|
|
|
return |
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
if not self.params.get('skip_download', False): |
|
|
|
if not self.params.get('skip_download', False): |
|
|
|
if self.params.get('nooverwrites', False) and os.path.exists(filename): |
|
|
|
if self.params.get('nooverwrites', False) and os.path.exists(_encodeFilename(filename)): |
|
|
|
success = True |
|
|
|
success = True |
|
|
|
else: |
|
|
|
else: |
|
|
|
try: |
|
|
|
try: |
|
|
@ -882,11 +891,11 @@ class FileDownloader(object): |
|
|
|
basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename] |
|
|
|
basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename] |
|
|
|
retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]) |
|
|
|
retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]) |
|
|
|
while retval == 2 or retval == 1: |
|
|
|
while retval == 2 or retval == 1: |
|
|
|
prevsize = os.path.getsize(tmpfilename) |
|
|
|
prevsize = os.path.getsize(_encodeFilename(tmpfilename)) |
|
|
|
self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) |
|
|
|
self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) |
|
|
|
time.sleep(5.0) # This seems to be needed |
|
|
|
time.sleep(5.0) # This seems to be needed |
|
|
|
retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) |
|
|
|
retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) |
|
|
|
cursize = os.path.getsize(tmpfilename) |
|
|
|
cursize = os.path.getsize(_encodeFilename(tmpfilename)) |
|
|
|
if prevsize == cursize and retval == 1: |
|
|
|
if prevsize == cursize and retval == 1: |
|
|
|
break |
|
|
|
break |
|
|
|
# Some rtmp streams seem abort after ~ 99.8%. Don't complain for those |
|
|
|
# Some rtmp streams seem abort after ~ 99.8%. Don't complain for those |
|
|
@ -895,7 +904,7 @@ class FileDownloader(object): |
|
|
|
retval = 0 |
|
|
|
retval = 0 |
|
|
|
break |
|
|
|
break |
|
|
|
if retval == 0: |
|
|
|
if retval == 0: |
|
|
|
self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename)) |
|
|
|
self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(_encodeFilename(tmpfilename))) |
|
|
|
self.try_rename(tmpfilename, filename) |
|
|
|
self.try_rename(tmpfilename, filename) |
|
|
|
return True |
|
|
|
return True |
|
|
|
else: |
|
|
|
else: |
|
|
@ -907,7 +916,7 @@ class FileDownloader(object): |
|
|
|
player_url = info_dict.get('player_url', None) |
|
|
|
player_url = info_dict.get('player_url', None) |
|
|
|
|
|
|
|
|
|
|
|
# Check file already present |
|
|
|
# Check file already present |
|
|
|
if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False): |
|
|
|
if self.params.get('continuedl', False) and os.path.isfile(_encodeFilename(filename)) and not self.params.get('nopart', False): |
|
|
|
self.report_file_already_downloaded(filename) |
|
|
|
self.report_file_already_downloaded(filename) |
|
|
|
return True |
|
|
|
return True |
|
|
|
|
|
|
|
|
|
|
@ -924,8 +933,8 @@ class FileDownloader(object): |
|
|
|
request = urllib2.Request(url, None, headers) |
|
|
|
request = urllib2.Request(url, None, headers) |
|
|
|
|
|
|
|
|
|
|
|
# Establish possible resume length |
|
|
|
# Establish possible resume length |
|
|
|
if os.path.isfile(tmpfilename): |
|
|
|
if os.path.isfile(_encodeFilename(tmpfilename)): |
|
|
|
resume_len = os.path.getsize(tmpfilename) |
|
|
|
resume_len = os.path.getsize(_encodeFilename(tmpfilename)) |
|
|
|
else: |
|
|
|
else: |
|
|
|
resume_len = 0 |
|
|
|
resume_len = 0 |
|
|
|
|
|
|
|
|
|
|
@ -4042,7 +4051,7 @@ class FFmpegExtractAudioPP(PostProcessor): |
|
|
|
@staticmethod |
|
|
|
@staticmethod |
|
|
|
def get_audio_codec(path): |
|
|
|
def get_audio_codec(path): |
|
|
|
try: |
|
|
|
try: |
|
|
|
cmd = ['ffprobe', '-show_streams', '--', path] |
|
|
|
cmd = ['ffprobe', '-show_streams', '--', _encodeFilename(path)] |
|
|
|
handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE) |
|
|
|
handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE) |
|
|
|
output = handle.communicate()[0] |
|
|
|
output = handle.communicate()[0] |
|
|
|
if handle.wait() != 0: |
|
|
|
if handle.wait() != 0: |
|
|
@ -4063,7 +4072,7 @@ class FFmpegExtractAudioPP(PostProcessor): |
|
|
|
acodec_opts = [] |
|
|
|
acodec_opts = [] |
|
|
|
else: |
|
|
|
else: |
|
|
|
acodec_opts = ['-acodec', codec] |
|
|
|
acodec_opts = ['-acodec', codec] |
|
|
|
cmd = ['ffmpeg', '-y', '-i', path, '-vn'] + acodec_opts + more_opts + ['--', out_path] |
|
|
|
cmd = ['ffmpeg', '-y', '-i', _encodeFilename(path), '-vn'] + acodec_opts + more_opts + ['--', _encodeFilename(out_path)] |
|
|
|
try: |
|
|
|
try: |
|
|
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
|
|
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
|
|
|
stdout,stderr = p.communicate() |
|
|
|
stdout,stderr = p.communicate() |
|
|
@ -4124,9 +4133,9 @@ class FFmpegExtractAudioPP(PostProcessor): |
|
|
|
extension = 'wav' |
|
|
|
extension = 'wav' |
|
|
|
more_opts += ['-f', 'wav'] |
|
|
|
more_opts += ['-f', 'wav'] |
|
|
|
|
|
|
|
|
|
|
|
(prefix, ext) = os.path.splitext(path) |
|
|
|
prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups |
|
|
|
new_path = prefix + '.' + extension |
|
|
|
new_path = prefix + sep + extension |
|
|
|
self._downloader.to_screen(u'[ffmpeg] Destination: %s' % new_path) |
|
|
|
self._downloader.to_screen(u'[ffmpeg] Destination: ' + new_path) |
|
|
|
try: |
|
|
|
try: |
|
|
|
self.run_ffmpeg(path, new_path, acodec, more_opts) |
|
|
|
self.run_ffmpeg(path, new_path, acodec, more_opts) |
|
|
|
except: |
|
|
|
except: |
|
|
@ -4140,13 +4149,13 @@ class FFmpegExtractAudioPP(PostProcessor): |
|
|
|
# Try to update the date time for extracted audio file. |
|
|
|
# Try to update the date time for extracted audio file. |
|
|
|
if information.get('filetime') is not None: |
|
|
|
if information.get('filetime') is not None: |
|
|
|
try: |
|
|
|
try: |
|
|
|
os.utime(new_path, (time.time(), information['filetime'])) |
|
|
|
os.utime(_encodeFilename(new_path), (time.time(), information['filetime'])) |
|
|
|
except: |
|
|
|
except: |
|
|
|
self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file') |
|
|
|
self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file') |
|
|
|
|
|
|
|
|
|
|
|
if not self._keepvideo: |
|
|
|
if not self._keepvideo: |
|
|
|
try: |
|
|
|
try: |
|
|
|
os.remove(path) |
|
|
|
os.remove(_encodeFilename(path)) |
|
|
|
except (IOError, OSError): |
|
|
|
except (IOError, OSError): |
|
|
|
self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file') |
|
|
|
self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file') |
|
|
|
return None |
|
|
|
return None |
|
|
@ -4194,9 +4203,9 @@ def parseOpts(): |
|
|
|
import optparse |
|
|
|
import optparse |
|
|
|
import shlex |
|
|
|
import shlex |
|
|
|
|
|
|
|
|
|
|
|
def _readOptions(filename): |
|
|
|
def _readOptions(filename_bytes): |
|
|
|
try: |
|
|
|
try: |
|
|
|
optionf = open(filename) |
|
|
|
optionf = open(filename_bytes) |
|
|
|
except IOError: |
|
|
|
except IOError: |
|
|
|
return [] # silently skip if file is not present |
|
|
|
return [] # silently skip if file is not present |
|
|
|
try: |
|
|
|
try: |
|
|
|