[archiveorg] improve format extraction(closes #10219)
parent
0ca057b965
commit
d50aca41f8
1 changed files with 38 additions and 40 deletions
@ -1,67 +1,65 @@ |
|||||||
from __future__ import unicode_literals |
from __future__ import unicode_literals |
||||||
|
|
||||||
from .common import InfoExtractor |
from .jwplatform import JWPlatformBaseIE |
||||||
from ..utils import unified_strdate |
from ..utils import ( |
||||||
|
unified_strdate, |
||||||
|
clean_html, |
||||||
|
) |
||||||
|
|
||||||
|
|
||||||
class ArchiveOrgIE(InfoExtractor): |
class ArchiveOrgIE(JWPlatformBaseIE): |
||||||
IE_NAME = 'archive.org' |
IE_NAME = 'archive.org' |
||||||
IE_DESC = 'archive.org videos' |
IE_DESC = 'archive.org videos' |
||||||
_VALID_URL = r'https?://(?:www\.)?archive\.org/details/(?P<id>[^?/]+)(?:[?].*)?$' |
_VALID_URL = r'https?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^/?#]+)(?:[?].*)?$' |
||||||
_TESTS = [{ |
_TESTS = [{ |
||||||
'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect', |
'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect', |
||||||
'md5': '8af1d4cf447933ed3c7f4871162602db', |
'md5': '8af1d4cf447933ed3c7f4871162602db', |
||||||
'info_dict': { |
'info_dict': { |
||||||
'id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect', |
'id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect', |
||||||
'ext': 'ogv', |
'ext': 'ogg', |
||||||
'title': '1968 Demo - FJCC Conference Presentation Reel #1', |
'title': '1968 Demo - FJCC Conference Presentation Reel #1', |
||||||
'description': 'md5:1780b464abaca9991d8968c877bb53ed', |
'description': 'md5:da45c349df039f1cc8075268eb1b5c25', |
||||||
'upload_date': '19681210', |
'upload_date': '19681210', |
||||||
'uploader': 'SRI International' |
'uploader': 'SRI International' |
||||||
} |
} |
||||||
}, { |
}, { |
||||||
'url': 'https://archive.org/details/Cops1922', |
'url': 'https://archive.org/details/Cops1922', |
||||||
'md5': '18f2a19e6d89af8425671da1cf3d4e04', |
'md5': 'bc73c8ab3838b5a8fc6c6651fa7b58ba', |
||||||
'info_dict': { |
'info_dict': { |
||||||
'id': 'Cops1922', |
'id': 'Cops1922', |
||||||
'ext': 'ogv', |
'ext': 'mp4', |
||||||
'title': 'Buster Keaton\'s "Cops" (1922)', |
'title': 'Buster Keaton\'s "Cops" (1922)', |
||||||
'description': 'md5:70f72ee70882f713d4578725461ffcc3', |
'description': 'md5:b4544662605877edd99df22f9620d858', |
||||||
} |
} |
||||||
|
}, { |
||||||
|
'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect', |
||||||
|
'only_matching': True, |
||||||
}] |
}] |
||||||
|
|
||||||
def _real_extract(self, url): |
def _real_extract(self, url): |
||||||
video_id = self._match_id(url) |
video_id = self._match_id(url) |
||||||
|
webpage = self._download_webpage( |
||||||
|
'http://archive.org/embed/' + video_id, video_id) |
||||||
|
jwplayer_playlist = self._parse_json(self._search_regex( |
||||||
|
r"(?s)Play\('[^']+'\s*,\s*(\[.+\])\s*,\s*{.*?}\);", |
||||||
|
webpage, 'jwplayer playlist'), video_id) |
||||||
|
info = self._parse_jwplayer_data( |
||||||
|
{'playlist': jwplayer_playlist}, video_id, base_url=url) |
||||||
|
|
||||||
json_url = url + ('&' if '?' in url else '?') + 'output=json' |
def get_optional(metadata, field): |
||||||
data = self._download_json(json_url, video_id) |
return metadata.get(field, [None])[0] |
||||||
|
|
||||||
def get_optional(data_dict, field): |
|
||||||
return data_dict['metadata'].get(field, [None])[0] |
|
||||||
|
|
||||||
title = get_optional(data, 'title') |
|
||||||
description = get_optional(data, 'description') |
|
||||||
uploader = get_optional(data, 'creator') |
|
||||||
upload_date = unified_strdate(get_optional(data, 'date')) |
|
||||||
|
|
||||||
formats = [ |
metadata = self._download_json( |
||||||
{ |
'http://archive.org/details/' + video_id, video_id, query={ |
||||||
'format': fdata['format'], |
'output': 'json', |
||||||
'url': 'http://' + data['server'] + data['dir'] + fn, |
})['metadata'] |
||||||
'file_size': int(fdata['size']), |
info.update({ |
||||||
} |
'title': get_optional(metadata, 'title') or info.get('title'), |
||||||
for fn, fdata in data['files'].items() |
'description': clean_html(get_optional(metadata, 'description')), |
||||||
if 'Video' in fdata['format']] |
}) |
||||||
|
if info.get('_type') != 'playlist': |
||||||
self._sort_formats(formats) |
info.update({ |
||||||
|
'uploader': get_optional(metadata, 'creator'), |
||||||
return { |
'upload_date': unified_strdate(get_optional(metadata, 'date')), |
||||||
'_type': 'video', |
}) |
||||||
'id': video_id, |
return info |
||||||
'title': title, |
|
||||||
'formats': formats, |
|
||||||
'description': description, |
|
||||||
'uploader': uploader, |
|
||||||
'upload_date': upload_date, |
|
||||||
'thumbnail': data.get('misc', {}).get('image'), |
|
||||||
} |
|
Loading…
Reference in new issue