commit
1b91a2e2cf
15 changed files with 4262 additions and 4249 deletions
@ -1,3 +1,5 @@ |
||||
*.pyc |
||||
*.pyo |
||||
*~ |
||||
wine-py2exe/ |
||||
py2exe.log |
||||
|
@ -0,0 +1,48 @@ |
||||
from distutils.core import setup |
||||
import py2exe |
||||
import sys, os |
||||
|
||||
"""This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package""" |
||||
|
||||
# If run without args, build executables |
||||
if len(sys.argv) == 1: |
||||
sys.argv.append("py2exe") |
||||
|
||||
# os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) # conflict with wine-py2exe.sh |
||||
sys.path.append('./youtube_dl') |
||||
|
||||
options = { |
||||
"bundle_files": 1, |
||||
"compressed": 1, |
||||
"optimize": 2, |
||||
"dist_dir": '.', |
||||
"dll_excludes": ['w9xpopen.exe'] |
||||
} |
||||
|
||||
console = [{ |
||||
"script":"./youtube_dl/__main__.py", |
||||
"dest_base": "youtube-dl", |
||||
}] |
||||
|
||||
init_file = open('./youtube_dl/__init__.py') |
||||
for line in init_file.readlines(): |
||||
if line.startswith('__version__'): |
||||
version = line[11:].strip(" ='\n") |
||||
break |
||||
else: |
||||
version = '' |
||||
|
||||
setup(name='youtube-dl', |
||||
version=version, |
||||
description='Small command-line program to download videos from YouTube.com and other video sites', |
||||
url='https://github.com/rg3/youtube-dl', |
||||
packages=['youtube_dl'], |
||||
|
||||
console = console, |
||||
options = {"py2exe": options}, |
||||
zipfile = None, |
||||
) |
||||
|
||||
import shutil |
||||
shutil.rmtree("build") |
||||
|
Binary file not shown.
Binary file not shown.
@ -0,0 +1,56 @@ |
||||
#!/bin/bash |
||||
|
||||
# Run with as parameter a setup.py that works in the current directory |
||||
# e.g. no os.chdir() |
||||
# It will run twice, the first time will crash |
||||
|
||||
set -e |
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" |
||||
|
||||
if [ ! -d wine-py2exe ]; then |
||||
|
||||
sudo apt-get install wine1.3 axel bsdiff |
||||
|
||||
mkdir wine-py2exe |
||||
cd wine-py2exe |
||||
export WINEPREFIX=`pwd` |
||||
|
||||
axel -a "http://www.python.org/ftp/python/2.7/python-2.7.msi" |
||||
axel -a "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe" |
||||
#axel -a "http://winetricks.org/winetricks" |
||||
|
||||
# http://appdb.winehq.org/objectManager.php?sClass=version&iId=21957 |
||||
echo "Follow python setup on screen" |
||||
wine msiexec /i python-2.7.msi |
||||
|
||||
echo "Follow py2exe setup on screen" |
||||
wine py2exe-0.6.9.win32-py2.7.exe |
||||
|
||||
#echo "Follow Microsoft Visual C++ 2008 Redistributable Package setup on screen" |
||||
#bash winetricks vcrun2008 |
||||
|
||||
rm py2exe-0.6.9.win32-py2.7.exe |
||||
rm python-2.7.msi |
||||
#rm winetricks |
||||
|
||||
# http://bugs.winehq.org/show_bug.cgi?id=3591 |
||||
|
||||
mv drive_c/Python27/Lib/site-packages/py2exe/run.exe drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup |
||||
bspatch drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run.exe "$SCRIPT_DIR/SizeOfImage.patch" |
||||
mv drive_c/Python27/Lib/site-packages/py2exe/run_w.exe drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup |
||||
bspatch drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run_w.exe "$SCRIPT_DIR/SizeOfImage_w.patch" |
||||
|
||||
cd - |
||||
|
||||
else |
||||
|
||||
export WINEPREFIX="$( cd wine-py2exe && pwd )" |
||||
|
||||
fi |
||||
|
||||
wine "C:\\Python27\\python.exe" "$1" py2exe > "py2exe.log" 2>&1 || true |
||||
echo '# Copying python27.dll' >> "py2exe.log" |
||||
cp "$WINEPREFIX/drive_c/windows/system32/python27.dll" build/bdist.win32/winexe/bundle-2.7/ |
||||
wine "C:\\Python27\\python.exe" "$1" py2exe >> "py2exe.log" 2>&1 |
||||
|
Binary file not shown.
Binary file not shown.
@ -0,0 +1,689 @@ |
||||
#!/usr/bin/env python |
||||
# -*- coding: utf-8 -*- |
||||
|
||||
import httplib |
||||
import math |
||||
import os |
||||
import re |
||||
import socket |
||||
import subprocess |
||||
import sys |
||||
import time |
||||
import urllib2 |
||||
|
||||
if os.name == 'nt': |
||||
import ctypes |
||||
|
||||
from utils import * |
||||
|
||||
|
||||
class FileDownloader(object): |
||||
"""File Downloader class. |
||||
|
||||
File downloader objects are the ones responsible of downloading the |
||||
actual video file and writing it to disk if the user has requested |
||||
it, among some other tasks. In most cases there should be one per |
||||
program. As, given a video URL, the downloader doesn't know how to |
||||
extract all the needed information, task that InfoExtractors do, it |
||||
has to pass the URL to one of them. |
||||
|
||||
For this, file downloader objects have a method that allows |
||||
InfoExtractors to be registered in a given order. When it is passed |
||||
a URL, the file downloader handles it to the first InfoExtractor it |
||||
finds that reports being able to handle it. The InfoExtractor extracts |
||||
all the information about the video or videos the URL refers to, and |
||||
asks the FileDownloader to process the video information, possibly |
||||
downloading the video. |
||||
|
||||
File downloaders accept a lot of parameters. In order not to saturate |
||||
the object constructor with arguments, it receives a dictionary of |
||||
options instead. These options are available through the params |
||||
attribute for the InfoExtractors to use. The FileDownloader also |
||||
registers itself as the downloader in charge for the InfoExtractors |
||||
that are added to it, so this is a "mutual registration". |
||||
|
||||
Available options: |
||||
|
||||
username: Username for authentication purposes. |
||||
password: Password for authentication purposes. |
||||
usenetrc: Use netrc for authentication instead. |
||||
quiet: Do not print messages to stdout. |
||||
forceurl: Force printing final URL. |
||||
forcetitle: Force printing title. |
||||
forcethumbnail: Force printing thumbnail URL. |
||||
forcedescription: Force printing description. |
||||
forcefilename: Force printing final filename. |
||||
simulate: Do not download the video files. |
||||
format: Video format code. |
||||
format_limit: Highest quality format to try. |
||||
outtmpl: Template for output names. |
||||
ignoreerrors: Do not stop on download errors. |
||||
ratelimit: Download speed limit, in bytes/sec. |
||||
nooverwrites: Prevent overwriting files. |
||||
retries: Number of times to retry for HTTP error 5xx |
||||
continuedl: Try to continue downloads if possible. |
||||
noprogress: Do not print the progress bar. |
||||
playliststart: Playlist item to start at. |
||||
playlistend: Playlist item to end at. |
||||
matchtitle: Download only matching titles. |
||||
rejecttitle: Reject downloads for matching titles. |
||||
logtostderr: Log messages to stderr instead of stdout. |
||||
consoletitle: Display progress in console window's titlebar. |
||||
nopart: Do not use temporary .part files. |
||||
updatetime: Use the Last-modified header to set output file timestamps. |
||||
writedescription: Write the video description to a .description file |
||||
writeinfojson: Write the video description to a .info.json file |
||||
writesubtitles: Write the video subtitles to a .srt file |
||||
subtitleslang: Language of the subtitles to download |
||||
""" |
||||
|
||||
params = None |
||||
_ies = [] |
||||
_pps = [] |
||||
_download_retcode = None |
||||
_num_downloads = None |
||||
_screen_file = None |
||||
|
||||
def __init__(self, params): |
||||
"""Create a FileDownloader object with the given options.""" |
||||
self._ies = [] |
||||
self._pps = [] |
||||
self._download_retcode = 0 |
||||
self._num_downloads = 0 |
||||
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] |
||||
self.params = params |
||||
|
||||
@staticmethod |
||||
def format_bytes(bytes): |
||||
if bytes is None: |
||||
return 'N/A' |
||||
if type(bytes) is str: |
||||
bytes = float(bytes) |
||||
if bytes == 0.0: |
||||
exponent = 0 |
||||
else: |
||||
exponent = long(math.log(bytes, 1024.0)) |
||||
suffix = 'bkMGTPEZY'[exponent] |
||||
converted = float(bytes) / float(1024 ** exponent) |
||||
return '%.2f%s' % (converted, suffix) |
||||
|
||||
@staticmethod |
||||
def calc_percent(byte_counter, data_len): |
||||
if data_len is None: |
||||
return '---.-%' |
||||
return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0)) |
||||
|
||||
@staticmethod |
||||
def calc_eta(start, now, total, current): |
||||
if total is None: |
||||
return '--:--' |
||||
dif = now - start |
||||
if current == 0 or dif < 0.001: # One millisecond |
||||
return '--:--' |
||||
rate = float(current) / dif |
||||
eta = long((float(total) - float(current)) / rate) |
||||
(eta_mins, eta_secs) = divmod(eta, 60) |
||||
if eta_mins > 99: |
||||
return '--:--' |
||||
return '%02d:%02d' % (eta_mins, eta_secs) |
||||
|
||||
@staticmethod |
||||
def calc_speed(start, now, bytes): |
||||
dif = now - start |
||||
if bytes == 0 or dif < 0.001: # One millisecond |
||||
return '%10s' % '---b/s' |
||||
return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif)) |
||||
|
||||
@staticmethod |
||||
def best_block_size(elapsed_time, bytes): |
||||
new_min = max(bytes / 2.0, 1.0) |
||||
new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB |
||||
if elapsed_time < 0.001: |
||||
return long(new_max) |
||||
rate = bytes / elapsed_time |
||||
if rate > new_max: |
||||
return long(new_max) |
||||
if rate < new_min: |
||||
return long(new_min) |
||||
return long(rate) |
||||
|
||||
@staticmethod |
||||
def parse_bytes(bytestr): |
||||
"""Parse a string indicating a byte quantity into a long integer.""" |
||||
matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) |
||||
if matchobj is None: |
||||
return None |
||||
number = float(matchobj.group(1)) |
||||
multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) |
||||
return long(round(number * multiplier)) |
||||
|
||||
def add_info_extractor(self, ie): |
||||
"""Add an InfoExtractor object to the end of the list.""" |
||||
self._ies.append(ie) |
||||
ie.set_downloader(self) |
||||
|
||||
def add_post_processor(self, pp): |
||||
"""Add a PostProcessor object to the end of the chain.""" |
||||
self._pps.append(pp) |
||||
pp.set_downloader(self) |
||||
|
||||
def to_screen(self, message, skip_eol=False): |
||||
"""Print message to stdout if not in quiet mode.""" |
||||
assert type(message) == type(u'') |
||||
if not self.params.get('quiet', False): |
||||
terminator = [u'\n', u''][skip_eol] |
||||
output = message + terminator |
||||
|
||||
if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr |
||||
output = output.encode(preferredencoding(), 'ignore') |
||||
self._screen_file.write(output) |
||||
self._screen_file.flush() |
||||
|
||||
def to_stderr(self, message): |
||||
"""Print message to stderr.""" |
||||
print >>sys.stderr, message.encode(preferredencoding()) |
||||
|
||||
def to_cons_title(self, message): |
||||
"""Set console/terminal window title to message.""" |
||||
if not self.params.get('consoletitle', False): |
||||
return |
||||
if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow(): |
||||
# c_wchar_p() might not be necessary if `message` is |
||||
# already of type unicode() |
||||
ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) |
||||
elif 'TERM' in os.environ: |
||||
sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding())) |
||||
|
||||
def fixed_template(self): |
||||
"""Checks if the output template is fixed.""" |
||||
return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None) |
||||
|
||||
def trouble(self, message=None): |
||||
"""Determine action to take when a download problem appears. |
||||
|
||||
Depending on if the downloader has been configured to ignore |
||||
download errors or not, this method may throw an exception or |
||||
not when errors are found, after printing the message. |
||||
""" |
||||
if message is not None: |
||||
self.to_stderr(message) |
||||
if not self.params.get('ignoreerrors', False): |
||||
raise DownloadError(message) |
||||
self._download_retcode = 1 |
||||
|
||||
def slow_down(self, start_time, byte_counter): |
||||
"""Sleep if the download speed is over the rate limit.""" |
||||
rate_limit = self.params.get('ratelimit', None) |
||||
if rate_limit is None or byte_counter == 0: |
||||
return |
||||
now = time.time() |
||||
elapsed = now - start_time |
||||
if elapsed <= 0.0: |
||||
return |
||||
speed = float(byte_counter) / elapsed |
||||
if speed > rate_limit: |
||||
time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) |
||||
|
||||
def temp_name(self, filename): |
||||
"""Returns a temporary filename for the given filename.""" |
||||
if self.params.get('nopart', False) or filename == u'-' or \ |
||||
(os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): |
||||
return filename |
||||
return filename + u'.part' |
||||
|
||||
def undo_temp_name(self, filename): |
||||
if filename.endswith(u'.part'): |
||||
return filename[:-len(u'.part')] |
||||
return filename |
||||
|
||||
def try_rename(self, old_filename, new_filename): |
||||
try: |
||||
if old_filename == new_filename: |
||||
return |
||||
os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) |
||||
except (IOError, OSError), err: |
||||
self.trouble(u'ERROR: unable to rename file') |
||||
|
||||
def try_utime(self, filename, last_modified_hdr): |
||||
"""Try to set the last-modified time of the given file.""" |
||||
if last_modified_hdr is None: |
||||
return |
||||
if not os.path.isfile(encodeFilename(filename)): |
||||
return |
||||
timestr = last_modified_hdr |
||||
if timestr is None: |
||||
return |
||||
filetime = timeconvert(timestr) |
||||
if filetime is None: |
||||
return filetime |
||||
try: |
||||
os.utime(filename, (time.time(), filetime)) |
||||
except: |
||||
pass |
||||
return filetime |
||||
|
||||
def report_writedescription(self, descfn): |
||||
""" Report that the description file is being written """ |
||||
self.to_screen(u'[info] Writing video description to: ' + descfn) |
||||
|
||||
def report_writesubtitles(self, srtfn): |
||||
""" Report that the subtitles file is being written """ |
||||
self.to_screen(u'[info] Writing video subtitles to: ' + srtfn) |
||||
|
||||
def report_writeinfojson(self, infofn): |
||||
""" Report that the metadata file has been written """ |
||||
self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) |
||||
|
||||
def report_destination(self, filename): |
||||
"""Report destination filename.""" |
||||
self.to_screen(u'[download] Destination: ' + filename) |
||||
|
||||
def report_progress(self, percent_str, data_len_str, speed_str, eta_str): |
||||
"""Report download progress.""" |
||||
if self.params.get('noprogress', False): |
||||
return |
||||
self.to_screen(u'\r[download] %s of %s at %s ETA %s' % |
||||
(percent_str, data_len_str, speed_str, eta_str), skip_eol=True) |
||||
self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' % |
||||
(percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) |
||||
|
||||
def report_resuming_byte(self, resume_len): |
||||
"""Report attempt to resume at given byte.""" |
||||
self.to_screen(u'[download] Resuming download at byte %s' % resume_len) |
||||
|
||||
def report_retry(self, count, retries): |
||||
"""Report retry in case of HTTP error 5xx""" |
||||
self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) |
||||
|
||||
def report_file_already_downloaded(self, file_name): |
||||
"""Report file has already been fully downloaded.""" |
||||
try: |
||||
self.to_screen(u'[download] %s has already been downloaded' % file_name) |
||||
except (UnicodeEncodeError), err: |
||||
self.to_screen(u'[download] The file has already been downloaded') |
||||
|
||||
def report_unable_to_resume(self): |
||||
"""Report it was impossible to resume download.""" |
||||
self.to_screen(u'[download] Unable to resume') |
||||
|
||||
def report_finish(self): |
||||
"""Report download finished.""" |
||||
if self.params.get('noprogress', False): |
||||
self.to_screen(u'[download] Download completed') |
||||
else: |
||||
self.to_screen(u'') |
||||
|
||||
def increment_downloads(self): |
||||
"""Increment the ordinal that assigns a number to each file.""" |
||||
self._num_downloads += 1 |
||||
|
||||
def prepare_filename(self, info_dict): |
||||
"""Generate the output filename.""" |
||||
try: |
||||
template_dict = dict(info_dict) |
||||
template_dict['epoch'] = unicode(long(time.time())) |
||||
template_dict['autonumber'] = unicode('%05d' % self._num_downloads) |
||||
filename = self.params['outtmpl'] % template_dict |
||||
return filename |
||||
except (ValueError, KeyError), err: |
||||
self.trouble(u'ERROR: invalid system charset or erroneous output template') |
||||
return None |
||||
|
||||
def _match_entry(self, info_dict): |
||||
""" Returns None iff the file should be downloaded """ |
||||
|
||||
title = info_dict['title'] |
||||
matchtitle = self.params.get('matchtitle', False) |
||||
if matchtitle and not re.search(matchtitle, title, re.IGNORECASE): |
||||
return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' |
||||
rejecttitle = self.params.get('rejecttitle', False) |
||||
if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE): |
||||
return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' |
||||
return None |
||||
|
||||
def process_info(self, info_dict): |
||||
"""Process a single dictionary returned by an InfoExtractor.""" |
||||
|
||||
info_dict['stitle'] = sanitize_filename(info_dict['title']) |
||||
|
||||
reason = self._match_entry(info_dict) |
||||
if reason is not None: |
||||
self.to_screen(u'[download] ' + reason) |
||||
return |
||||
|
||||
max_downloads = self.params.get('max_downloads') |
||||
if max_downloads is not None: |
||||
if self._num_downloads > int(max_downloads): |
||||
raise MaxDownloadsReached() |
||||
|
||||
filename = self.prepare_filename(info_dict) |
||||
|
||||
# Forced printings |
||||
if self.params.get('forcetitle', False): |
||||
print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace') |
||||
if self.params.get('forceurl', False): |
||||
print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace') |
||||
if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict: |
||||
print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace') |
||||
if self.params.get('forcedescription', False) and 'description' in info_dict: |
||||
print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace') |
||||
if self.params.get('forcefilename', False) and filename is not None: |
||||
print filename.encode(preferredencoding(), 'xmlcharrefreplace') |
||||
if self.params.get('forceformat', False): |
||||
print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace') |
||||
|
||||
# Do nothing else if in simulate mode |
||||
if self.params.get('simulate', False): |
||||
return |
||||
|
||||
if filename is None: |
||||
return |
||||
|
||||
try: |
||||
dn = os.path.dirname(encodeFilename(filename)) |
||||
if dn != '' and not os.path.exists(dn): # dn is already encoded |
||||
os.makedirs(dn) |
||||
except (OSError, IOError), err: |
||||
self.trouble(u'ERROR: unable to create directory ' + unicode(err)) |
||||
return |
||||
|
||||
if self.params.get('writedescription', False): |
||||
try: |
||||
descfn = filename + u'.description' |
||||
self.report_writedescription(descfn) |
||||
descfile = open(encodeFilename(descfn), 'wb') |
||||
try: |
||||
descfile.write(info_dict['description'].encode('utf-8')) |
||||
finally: |
||||
descfile.close() |
||||
except (OSError, IOError): |
||||
self.trouble(u'ERROR: Cannot write description file ' + descfn) |
||||
return |
||||
|
||||
if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']: |
||||
# subtitles download errors are already managed as troubles in relevant IE |
||||
# that way it will silently go on when used with unsupporting IE |
||||
try: |
||||
srtfn = filename.rsplit('.', 1)[0] + u'.srt' |
||||
self.report_writesubtitles(srtfn) |
||||
srtfile = open(encodeFilename(srtfn), 'wb') |
||||
try: |
||||
srtfile.write(info_dict['subtitles'].encode('utf-8')) |
||||
finally: |
||||
srtfile.close() |
||||
except (OSError, IOError): |
||||
self.trouble(u'ERROR: Cannot write subtitles file ' + descfn) |
||||
return |
||||
|
||||
if self.params.get('writeinfojson', False): |
||||
infofn = filename + u'.info.json' |
||||
self.report_writeinfojson(infofn) |
||||
try: |
||||
json.dump |
||||
except (NameError,AttributeError): |
||||
self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.') |
||||
return |
||||
try: |
||||
infof = open(encodeFilename(infofn), 'wb') |
||||
try: |
||||
json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',)) |
||||
json.dump(json_info_dict, infof) |
||||
finally: |
||||
infof.close() |
||||
except (OSError, IOError): |
||||
self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn) |
||||
return |
||||
|
||||
if not self.params.get('skip_download', False): |
||||
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)): |
||||
success = True |
||||
else: |
||||
try: |
||||
success = self._do_download(filename, info_dict) |
||||
except (OSError, IOError), err: |
||||
raise UnavailableVideoError |
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err: |
||||
self.trouble(u'ERROR: unable to download video data: %s' % str(err)) |
||||
return |
||||
except (ContentTooShortError, ), err: |
||||
self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) |
||||
return |
||||
|
||||
if success: |
||||
try: |
||||
self.post_process(filename, info_dict) |
||||
except (PostProcessingError), err: |
||||
self.trouble(u'ERROR: postprocessing: %s' % str(err)) |
||||
return |
||||
|
||||
def download(self, url_list): |
||||
"""Download a given list of URLs.""" |
||||
if len(url_list) > 1 and self.fixed_template(): |
||||
raise SameFileError(self.params['outtmpl']) |
||||
|
||||
for url in url_list: |
||||
suitable_found = False |
||||
for ie in self._ies: |
||||
# Go to next InfoExtractor if not suitable |
||||
if not ie.suitable(url): |
||||
continue |
||||
|
||||
# Suitable InfoExtractor found |
||||
suitable_found = True |
||||
|
||||
# Extract information from URL and process it |
||||
videos = ie.extract(url) |
||||
for video in videos or []: |
||||
try: |
||||
self.increment_downloads() |
||||
self.process_info(video) |
||||
except UnavailableVideoError: |
||||
self.trouble(u'\nERROR: unable to download video') |
||||
|
||||
# Suitable InfoExtractor had been found; go to next URL |
||||
break |
||||
|
||||
if not suitable_found: |
||||
self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) |
||||
|
||||
return self._download_retcode |
||||
|
||||
def post_process(self, filename, ie_info): |
||||
"""Run the postprocessing chain on the given file.""" |
||||
info = dict(ie_info) |
||||
info['filepath'] = filename |
||||
for pp in self._pps: |
||||
info = pp.run(info) |
||||
if info is None: |
||||
break |
||||
|
||||
def _download_with_rtmpdump(self, filename, url, player_url): |
||||
self.report_destination(filename) |
||||
tmpfilename = self.temp_name(filename) |
||||
|
||||
# Check for rtmpdump first |
||||
try: |
||||
subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT) |
||||
except (OSError, IOError): |
||||
self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run') |
||||
return False |
||||
|
||||
# Download using rtmpdump. rtmpdump returns exit code 2 when |
||||
# the connection was interrumpted and resuming appears to be |
||||
# possible. This is part of rtmpdump's normal usage, AFAIK. |
||||
basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename] |
||||
args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)] |
||||
if self.params.get('verbose', False): |
||||
try: |
||||
import pipes |
||||
shell_quote = lambda args: ' '.join(map(pipes.quote, args)) |
||||
except ImportError: |
||||
shell_quote = repr |
||||
self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args)) |
||||
retval = subprocess.call(args) |
||||
while retval == 2 or retval == 1: |
||||
prevsize = os.path.getsize(encodeFilename(tmpfilename)) |
||||
self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) |
||||
time.sleep(5.0) # This seems to be needed |
||||
retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) |
||||
cursize = os.path.getsize(encodeFilename(tmpfilename)) |
||||
if prevsize == cursize and retval == 1: |
||||
break |
||||
# Some rtmp streams seem abort after ~ 99.8%. Don't complain for those |
||||
if prevsize == cursize and retval == 2 and cursize > 1024: |
||||
self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.') |
||||
retval = 0 |
||||
break |
||||
if retval == 0: |
||||
self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename))) |
||||
self.try_rename(tmpfilename, filename) |
||||
return True |
||||
else: |
||||
self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval) |
||||
return False |
||||
|
||||
def _do_download(self, filename, info_dict): |
||||
url = info_dict['url'] |
||||
player_url = info_dict.get('player_url', None) |
||||
|
||||
# Check file already present |
||||
if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False): |
||||
self.report_file_already_downloaded(filename) |
||||
return True |
||||
|
||||
# Attempt to download using rtmpdump |
||||
if url.startswith('rtmp'): |
||||
return self._download_with_rtmpdump(filename, url, player_url) |
||||
|
||||
tmpfilename = self.temp_name(filename) |
||||
stream = None |
||||
|
||||
# Do not include the Accept-Encoding header |
||||
headers = {'Youtubedl-no-compression': 'True'} |
||||
basic_request = urllib2.Request(url, None, headers) |
||||
request = urllib2.Request(url, None, headers) |
||||
|
||||
# Establish possible resume length |
||||
if os.path.isfile(encodeFilename(tmpfilename)): |
||||
resume_len = os.path.getsize(encodeFilename(tmpfilename)) |
||||
else: |
||||
resume_len = 0 |
||||
|
||||
open_mode = 'wb' |
||||
if resume_len != 0: |
||||
if self.params.get('continuedl', False): |
||||
self.report_resuming_byte(resume_len) |
||||
request.add_header('Range','bytes=%d-' % resume_len) |
||||
open_mode = 'ab' |
||||
else: |
||||
resume_len = 0 |
||||
|
||||
count = 0 |
||||
retries = self.params.get('retries', 0) |
||||
while count <= retries: |
||||
# Establish connection |
||||
try: |
||||
if count == 0 and 'urlhandle' in info_dict: |
||||
data = info_dict['urlhandle'] |
||||
data = urllib2.urlopen(request) |
||||
break |
||||
except (urllib2.HTTPError, ), err: |
||||
if (err.code < 500 or err.code >= 600) and err.code != 416: |
||||
# Unexpected HTTP error |
||||
raise |
||||
elif err.code == 416: |
||||
# Unable to resume (requested range not satisfiable) |
||||
try: |
||||
# Open the connection again without the range header |
||||
data = urllib2.urlopen(basic_request) |
||||
content_length = data.info()['Content-Length'] |
||||
except (urllib2.HTTPError, ), err: |
||||
if err.code < 500 or err.code >= 600: |
||||
raise |
||||
else: |
||||
# Examine the reported length |
||||
if (content_length is not None and |
||||
(resume_len - 100 < long(content_length) < resume_len + 100)): |
||||
# The file had already been fully downloaded. |
||||
# Explanation to the above condition: in issue #175 it was revealed that |
||||
# YouTube sometimes adds or removes a few bytes from the end of the file, |
||||
# changing the file size slightly and causing problems for some users. So |
||||
# I decided to implement a suggested change and consider the file |
||||
# completely downloaded if the file size differs less than 100 bytes from |
||||
# the one in the hard drive. |
||||
self.report_file_already_downloaded(filename) |
||||
self.try_rename(tmpfilename, filename) |
||||
return True |
||||
else: |
||||
# The length does not match, we start the download over |
||||
self.report_unable_to_resume() |
||||
open_mode = 'wb' |
||||
break |
||||
# Retry |
||||
count += 1 |
||||
if count <= retries: |
||||
self.report_retry(count, retries) |
||||
|
||||
if count > retries: |
||||
self.trouble(u'ERROR: giving up after %s retries' % retries) |
||||
return False |
||||
|
||||
data_len = data.info().get('Content-length', None) |
||||
if data_len is not None: |
||||
data_len = long(data_len) + resume_len |
||||
data_len_str = self.format_bytes(data_len) |
||||
byte_counter = 0 + resume_len |
||||
block_size = 1024 |
||||
start = time.time() |
||||
while True: |
||||
# Download and write |
||||
before = time.time() |
||||
data_block = data.read(block_size) |
||||
after = time.time() |
||||
if len(data_block) == 0: |
||||
break |
||||
byte_counter += len(data_block) |
||||
|
||||
# Open file just in time |
||||
if stream is None: |
||||
try: |
||||
(stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) |
||||
assert stream is not None |
||||
filename = self.undo_temp_name(tmpfilename) |
||||
self.report_destination(filename) |
||||
except (OSError, IOError), err: |
||||
self.trouble(u'ERROR: unable to open for writing: %s' % str(err)) |
||||
return False |
||||
try: |
||||
stream.write(data_block) |
||||
except (IOError, OSError), err: |
||||
self.trouble(u'\nERROR: unable to write data: %s' % str(err)) |
||||
return False |
||||
block_size = self.best_block_size(after - before, len(data_block)) |
||||
|
||||
# Progress message |
||||
speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len) |
||||
if data_len is None: |
||||
self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA') |
||||
else: |
||||
percent_str = self.calc_percent(byte_counter, data_len) |
||||
eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len) |
||||
self.report_progress(percent_str, data_len_str, speed_str, eta_str) |
||||
|
||||
# Apply rate limit |
||||
self.slow_down(start, byte_counter - resume_len) |
||||
|
||||
if stream is None: |
||||
self.trouble(u'\nERROR: Did not get any data blocks') |
||||
return False |
||||
stream.close() |
||||
self.report_finish() |
||||
if data_len is not None and byte_counter != data_len: |
||||
raise ContentTooShortError(byte_counter, long(data_len)) |
||||
self.try_rename(tmpfilename, filename) |
||||
|
||||
# Update file modification time |
||||
if self.params.get('updatetime', True): |
||||
info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None)) |
||||
|
||||
return True |
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,185 @@ |
||||
#!/usr/bin/env python |
||||
# -*- coding: utf-8 -*- |
||||
|
||||
import os |
||||
import subprocess |
||||
import sys |
||||
import time |
||||
|
||||
from utils import * |
||||
|
||||
|
||||
class PostProcessor(object): |
||||
"""Post Processor class. |
||||
|
||||
PostProcessor objects can be added to downloaders with their |
||||
add_post_processor() method. When the downloader has finished a |
||||
successful download, it will take its internal chain of PostProcessors |
||||
and start calling the run() method on each one of them, first with |
||||
an initial argument and then with the returned value of the previous |
||||
PostProcessor. |
||||
|
||||
The chain will be stopped if one of them ever returns None or the end |
||||
of the chain is reached. |
||||
|
||||
PostProcessor objects follow a "mutual registration" process similar |
||||
to InfoExtractor objects. |
||||
""" |
||||
|
||||
_downloader = None |
||||
|
||||
def __init__(self, downloader=None): |
||||
self._downloader = downloader |
||||
|
||||
def set_downloader(self, downloader): |
||||
"""Sets the downloader for this PP.""" |
||||
self._downloader = downloader |
||||
|
||||
def run(self, information): |
||||
"""Run the PostProcessor. |
||||
|
||||
The "information" argument is a dictionary like the ones |
||||
composed by InfoExtractors. The only difference is that this |
||||
one has an extra field called "filepath" that points to the |
||||
downloaded file. |
||||
|
||||
When this method returns None, the postprocessing chain is |
||||
stopped. However, this method may return an information |
||||
dictionary that will be passed to the next postprocessing |
||||
object in the chain. It can be the one it received after |
||||
changing some fields. |
||||
|
||||
In addition, this method may raise a PostProcessingError |
||||
exception that will be taken into account by the downloader |
||||
it was called from. |
||||
""" |
||||
return information # by default, do nothing |
||||
|
||||
class AudioConversionError(BaseException): |
||||
def __init__(self, message): |
||||
self.message = message |
||||
|
||||
class FFmpegExtractAudioPP(PostProcessor): |
||||
|
||||
def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False): |
||||
PostProcessor.__init__(self, downloader) |
||||
if preferredcodec is None: |
||||
preferredcodec = 'best' |
||||
self._preferredcodec = preferredcodec |
||||
self._preferredquality = preferredquality |
||||
self._keepvideo = keepvideo |
||||
|
||||
@staticmethod |
||||
def get_audio_codec(path): |
||||
try: |
||||
cmd = ['ffprobe', '-show_streams', '--', encodeFilename(path)] |
||||
handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE) |
||||
output = handle.communicate()[0] |
||||
if handle.wait() != 0: |
||||
return None |
||||
except (IOError, OSError): |
||||
return None |
||||
audio_codec = None |
||||
for line in output.split('\n'): |
||||
if line.startswith('codec_name='): |
||||
audio_codec = line.split('=')[1].strip() |
||||
elif line.strip() == 'codec_type=audio' and audio_codec is not None: |
||||
return audio_codec |
||||
return None |
||||
|
||||
@staticmethod |
||||
def run_ffmpeg(path, out_path, codec, more_opts): |
||||
if codec is None: |
||||
acodec_opts = [] |
||||
else: |
||||
acodec_opts = ['-acodec', codec] |
||||
cmd = ['ffmpeg', '-y', '-i', encodeFilename(path), '-vn'] + acodec_opts + more_opts + ['--', encodeFilename(out_path)] |
||||
try: |
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
||||
stdout,stderr = p.communicate() |
||||
except (IOError, OSError): |
||||
e = sys.exc_info()[1] |
||||
if isinstance(e, OSError) and e.errno == 2: |
||||
raise AudioConversionError('ffmpeg not found. Please install ffmpeg.') |
||||
else: |
||||
raise e |
||||
if p.returncode != 0: |
||||
msg = stderr.strip().split('\n')[-1] |
||||
raise AudioConversionError(msg) |
||||
|
||||
def run(self, information): |
||||
path = information['filepath'] |
||||
|
||||
filecodec = self.get_audio_codec(path) |
||||
if filecodec is None: |
||||
self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe') |
||||
return None |
||||
|
||||
more_opts = [] |
||||
if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'): |
||||
if self._preferredcodec == 'm4a' and filecodec == 'aac': |
||||
# Lossless, but in another container |
||||
acodec = 'copy' |
||||
extension = self._preferredcodec |
||||
more_opts = ['-absf', 'aac_adtstoasc'] |
||||
elif filecodec in ['aac', 'mp3', 'vorbis']: |
||||
# Lossless if possible |
||||
acodec = 'copy' |
||||
extension = filecodec |
||||
if filecodec == 'aac': |
||||
more_opts = ['-f', 'adts'] |
||||
if filecodec == 'vorbis': |
||||
extension = 'ogg' |
||||
else: |
||||
# MP3 otherwise. |
||||
acodec = 'libmp3lame' |
||||
extension = 'mp3' |
||||
more_opts = [] |
||||
if self._preferredquality is not None: |
||||
more_opts += ['-ab', self._preferredquality] |
||||
else: |
||||
# We convert the audio (lossy) |
||||
acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec] |
||||
extension = self._preferredcodec |
||||
more_opts = [] |
||||
if self._preferredquality is not None: |
||||
more_opts += ['-ab', self._preferredquality] |
||||
if self._preferredcodec == 'aac': |
||||
more_opts += ['-f', 'adts'] |
||||
if self._preferredcodec == 'm4a': |
||||
more_opts += ['-absf', 'aac_adtstoasc'] |
||||
if self._preferredcodec == 'vorbis': |
||||
extension = 'ogg' |
||||
if self._preferredcodec == 'wav': |
||||
extension = 'wav' |
||||
more_opts += ['-f', 'wav'] |
||||
|
||||
prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups |
||||
new_path = prefix + sep + extension |
||||
self._downloader.to_screen(u'[ffmpeg] Destination: ' + new_path) |
||||
try: |
||||
self.run_ffmpeg(path, new_path, acodec, more_opts) |
||||
except: |
||||
etype,e,tb = sys.exc_info() |
||||
if isinstance(e, AudioConversionError): |
||||
self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message) |
||||
else: |
||||
self._downloader.to_stderr(u'ERROR: error running ffmpeg') |
||||
return None |
||||
|
||||
# Try to update the date time for extracted audio file. |
||||
if information.get('filetime') is not None: |
||||
try: |
||||
os.utime(encodeFilename(new_path), (time.time(), information['filetime'])) |
||||
except: |
||||
self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file') |
||||
|
||||
if not self._keepvideo: |
||||
try: |
||||
os.remove(encodeFilename(path)) |
||||
except (IOError, OSError): |
||||
self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file') |
||||
return None |
||||
|
||||
information['filepath'] = new_path |
||||
return information |
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,7 @@ |
||||
#!/usr/bin/env python |
||||
# -*- coding: utf-8 -*- |
||||
|
||||
import __init__ |
||||
|
||||
if __name__ == '__main__': |
||||
__init__.main() |
@ -0,0 +1,354 @@ |
||||
#!/usr/bin/env python |
||||
# -*- coding: utf-8 -*- |
||||
|
||||
import gzip |
||||
import htmlentitydefs |
||||
import HTMLParser |
||||
import locale |
||||
import os |
||||
import re |
||||
import sys |
||||
import zlib |
||||
import urllib2 |
||||
import email.utils |
||||
import json |
||||
|
||||
try: |
||||
import cStringIO as StringIO |
||||
except ImportError: |
||||
import StringIO |
||||
|
||||
std_headers = { |
||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1', |
||||
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', |
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', |
||||
'Accept-Encoding': 'gzip, deflate', |
||||
'Accept-Language': 'en-us,en;q=0.5', |
||||
} |
||||
|
||||
def preferredencoding(): |
||||
"""Get preferred encoding. |
||||
|
||||
Returns the best encoding scheme for the system, based on |
||||
locale.getpreferredencoding() and some further tweaks. |
||||
""" |
||||
def yield_preferredencoding(): |
||||
try: |
||||
pref = locale.getpreferredencoding() |
||||
u'TEST'.encode(pref) |
||||
except: |
||||
pref = 'UTF-8' |
||||
while True: |
||||
yield pref |
||||
return yield_preferredencoding().next() |
||||
|
||||
|
||||
def htmlentity_transform(matchobj): |
||||
"""Transforms an HTML entity to a Unicode character. |
||||
|
||||
This function receives a match object and is intended to be used with |
||||
the re.sub() function. |
||||
""" |
||||
entity = matchobj.group(1) |
||||
|
||||
# Known non-numeric HTML entity |
||||
if entity in htmlentitydefs.name2codepoint: |
||||
return unichr(htmlentitydefs.name2codepoint[entity]) |
||||
|
||||
# Unicode character |
||||
mobj = re.match(ur'(?u)#(x?\d+)', entity) |
||||
if mobj is not None: |
||||
numstr = mobj.group(1) |
||||
if numstr.startswith(u'x'): |
||||
base = 16 |
||||
numstr = u'0%s' % numstr |
||||
else: |
||||
base = 10 |
||||
return unichr(long(numstr, base)) |
||||
|
||||
# Unknown entity in name, return its literal representation |
||||
return (u'&%s;' % entity) |
||||
|
||||
HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix |
||||
class IDParser(HTMLParser.HTMLParser): |
||||
"""Modified HTMLParser that isolates a tag with the specified id""" |
||||
def __init__(self, id): |
||||
self.id = id |
||||
self.result = None |
||||
self.started = False |
||||
self.depth = {} |
||||
self.html = None |
||||
self.watch_startpos = False |
||||
self.error_count = 0 |
||||
HTMLParser.HTMLParser.__init__(self) |
||||
|
||||
def error(self, message): |
||||
print self.getpos() |
||||
if self.error_count > 10 or self.started: |
||||
raise HTMLParser.HTMLParseError(message, self.getpos()) |
||||
self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line |
||||
self.error_count += 1 |
||||
self.goahead(1) |
||||
|
||||
def loads(self, html): |
||||
self.html = html |
||||
self.feed(html) |
||||
self.close() |
||||
|
||||
def handle_starttag(self, tag, attrs): |
||||
attrs = dict(attrs) |
||||
if self.started: |
||||
self.find_startpos(None) |
||||
if 'id' in attrs and attrs['id'] == self.id: |
||||
self.result = [tag] |
||||
self.started = True |
||||
self.watch_startpos = True |
||||
if self.started: |
||||
if not tag in self.depth: self.depth[tag] = 0 |
||||
self.depth[tag] += 1 |
||||
|
||||
def handle_endtag(self, tag): |
||||
if self.started: |
||||
if tag in self.depth: self.depth[tag] -= 1 |
||||
if self.depth[self.result[0]] == 0: |
||||
self.started = False |
||||
self.result.append(self.getpos()) |
||||
|
||||
def find_startpos(self, x): |
||||
"""Needed to put the start position of the result (self.result[1]) |
||||
after the opening tag with the requested id""" |
||||
if self.watch_startpos: |
||||
self.watch_startpos = False |
||||
self.result.append(self.getpos()) |
||||
handle_entityref = handle_charref = handle_data = handle_comment = \ |
||||
handle_decl = handle_pi = unknown_decl = find_startpos |
||||
|
||||
def get_result(self): |
||||
if self.result == None: return None |
||||
if len(self.result) != 3: return None |
||||
lines = self.html.split('\n') |
||||
lines = lines[self.result[1][0]-1:self.result[2][0]] |
||||
lines[0] = lines[0][self.result[1][1]:] |
||||
if len(lines) == 1: |
||||
lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]] |
||||
lines[-1] = lines[-1][:self.result[2][1]] |
||||
return '\n'.join(lines).strip() |
||||
|
||||
def get_element_by_id(id, html): |
||||
"""Return the content of the tag with the specified id in the passed HTML document""" |
||||
parser = IDParser(id) |
||||
try: |
||||
parser.loads(html) |
||||
except HTMLParser.HTMLParseError: |
||||
pass |
||||
return parser.get_result() |
||||
|
||||
|
||||
def clean_html(html): |
||||
"""Clean an HTML snippet into a readable string""" |
||||
# Newline vs <br /> |
||||
html = html.replace('\n', ' ') |
||||
html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html) |
||||
# Strip html tags |
||||
html = re.sub('<.*?>', '', html) |
||||
# Replace html entities |
||||
html = unescapeHTML(html) |
||||
return html |
||||
|
||||
|
||||
def sanitize_open(filename, open_mode): |
||||
"""Try to open the given filename, and slightly tweak it if this fails. |
||||
|
||||
Attempts to open the given filename. If this fails, it tries to change |
||||
the filename slightly, step by step, until it's either able to open it |
||||
or it fails and raises a final exception, like the standard open() |
||||
function. |
||||
|
||||
It returns the tuple (stream, definitive_file_name). |
||||
""" |
||||
try: |
||||
if filename == u'-': |
||||
if sys.platform == 'win32': |
||||
import msvcrt |
||||
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) |
||||
return (sys.stdout, filename) |
||||
stream = open(encodeFilename(filename), open_mode) |
||||
return (stream, filename) |
||||
except (IOError, OSError), err: |
||||
# In case of error, try to remove win32 forbidden chars |
||||
filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename) |
||||
|
||||
# An exception here should be caught in the caller |
||||
stream = open(encodeFilename(filename), open_mode) |
||||
return (stream, filename) |
||||
|
||||
|
||||
def timeconvert(timestr): |
||||
"""Convert RFC 2822 defined time string into system timestamp""" |
||||
timestamp = None |
||||
timetuple = email.utils.parsedate_tz(timestr) |
||||
if timetuple is not None: |
||||
timestamp = email.utils.mktime_tz(timetuple) |
||||
return timestamp |
||||
|
||||
def sanitize_filename(s): |
||||
"""Sanitizes a string so it could be used as part of a filename.""" |
||||
def replace_insane(char): |
||||
if char in u' .\\/|?*<>:"' or ord(char) < 32: |
||||
return '_' |
||||
return char |
||||
return u''.join(map(replace_insane, s)).strip('_') |
||||
|
||||
def orderedSet(iterable): |
||||
""" Remove all duplicates from the input iterable """ |
||||
res = [] |
||||
for el in iterable: |
||||
if el not in res: |
||||
res.append(el) |
||||
return res |
||||
|
||||
def unescapeHTML(s): |
||||
""" |
||||
@param s a string (of type unicode) |
||||
""" |
||||
assert type(s) == type(u'') |
||||
|
||||
result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s) |
||||
return result |
||||
|
||||
def encodeFilename(s): |
||||
""" |
||||
@param s The name of the file (of type unicode) |
||||
""" |
||||
|
||||
assert type(s) == type(u'') |
||||
|
||||
if sys.platform == 'win32' and sys.getwindowsversion().major >= 5: |
||||
# Pass u'' directly to use Unicode APIs on Windows 2000 and up |
||||
# (Detecting Windows NT 4 is tricky because 'major >= 4' would |
||||
# match Windows 9x series as well. Besides, NT 4 is obsolete.) |
||||
return s |
||||
else: |
||||
return s.encode(sys.getfilesystemencoding(), 'ignore') |
||||
|
||||
class DownloadError(Exception): |
||||
"""Download Error exception. |
||||
|
||||
This exception may be thrown by FileDownloader objects if they are not |
||||
configured to continue on errors. They will contain the appropriate |
||||
error message. |
||||
""" |
||||
pass |
||||
|
||||
|
||||
class SameFileError(Exception): |
||||
"""Same File exception. |
||||
|
||||
This exception will be thrown by FileDownloader objects if they detect |
||||
multiple files would have to be downloaded to the same file on disk. |
||||
""" |
||||
pass |
||||
|
||||
|
||||
class PostProcessingError(Exception): |
||||
"""Post Processing exception. |
||||
|
||||
This exception may be raised by PostProcessor's .run() method to |
||||
indicate an error in the postprocessing task. |
||||
""" |
||||
pass |
||||
|
||||
class MaxDownloadsReached(Exception): |
||||
""" --max-downloads limit has been reached. """ |
||||
pass |
||||
|
||||
|
||||
class UnavailableVideoError(Exception): |
||||
"""Unavailable Format exception. |
||||
|
||||
This exception will be thrown when a video is requested |
||||
in a format that is not available for that video. |
||||
""" |
||||
pass |
||||
|
||||
|
||||
class ContentTooShortError(Exception): |
||||
"""Content Too Short exception. |
||||
|
||||
This exception may be raised by FileDownloader objects when a file they |
||||
download is too small for what the server announced first, indicating |
||||
the connection was probably interrupted. |
||||
""" |
||||
# Both in bytes |
||||
downloaded = None |
||||
expected = None |
||||
|
||||
def __init__(self, downloaded, expected): |
||||
self.downloaded = downloaded |
||||
self.expected = expected |
||||
|
||||
|
||||
class Trouble(Exception): |
||||
"""Trouble helper exception |
||||
|
||||
This is an exception to be handled with |
||||
FileDownloader.trouble |
||||
""" |
||||
|
||||
class YoutubeDLHandler(urllib2.HTTPHandler): |
||||
"""Handler for HTTP requests and responses. |
||||
|
||||
This class, when installed with an OpenerDirector, automatically adds |
||||
the standard headers to every HTTP request and handles gzipped and |
||||
deflated responses from web servers. If compression is to be avoided in |
||||
a particular request, the original request in the program code only has |
||||
to include the HTTP header "Youtubedl-No-Compression", which will be |
||||
removed before making the real request. |
||||
|
||||
Part of this code was copied from: |
||||
|
||||
http://techknack.net/python-urllib2-handlers/ |
||||
|
||||
Andrew Rowls, the author of that code, agreed to release it to the |
||||
public domain. |
||||
""" |
||||
|
||||
@staticmethod |
||||
def deflate(data): |
||||
try: |
||||
return zlib.decompress(data, -zlib.MAX_WBITS) |
||||
except zlib.error: |
||||
return zlib.decompress(data) |
||||
|
||||
@staticmethod |
||||
def addinfourl_wrapper(stream, headers, url, code): |
||||
if hasattr(urllib2.addinfourl, 'getcode'): |
||||
return urllib2.addinfourl(stream, headers, url, code) |
||||
ret = urllib2.addinfourl(stream, headers, url) |
||||
ret.code = code |
||||
return ret |
||||
|
||||
def http_request(self, req): |
||||
for h in std_headers: |
||||
if h in req.headers: |
||||
del req.headers[h] |
||||
req.add_header(h, std_headers[h]) |
||||
if 'Youtubedl-no-compression' in req.headers: |
||||
if 'Accept-encoding' in req.headers: |
||||
del req.headers['Accept-encoding'] |
||||
del req.headers['Youtubedl-no-compression'] |
||||
return req |
||||
|
||||
def http_response(self, req, resp): |
||||
old_resp = resp |
||||
# gzip |
||||
if resp.headers.get('Content-encoding', '') == 'gzip': |
||||
gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r') |
||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) |
||||
resp.msg = old_resp.msg |
||||
# deflate |
||||
if resp.headers.get('Content-encoding', '') == 'deflate': |
||||
gz = StringIO.StringIO(self.deflate(resp.read())) |
||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) |
||||
resp.msg = old_resp.msg |
||||
return resp |
Loading…
Reference in new issue