Update youtube-dl v2021.01.03

Author: oleksis

youtube_dl/extractor/acast.py

@@ -2,21 +2,47 @@
 from __future__ import unicode_literals
 
 import re
-import functools
 
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     clean_html,
-    float_or_none,
     int_or_none,
-    try_get,
-    unified_timestamp,
-    OnDemandPagedList,
+    parse_iso8601,
 )
 
 
-class ACastIE(InfoExtractor):
+class ACastBaseIE(InfoExtractor):
+    def _extract_episode(self, episode, show_info):
+        title = episode['title']
+        info = {
+            'id': episode['id'],
+            'display_id': episode.get('episodeUrl'),
+            'url': episode['url'],
+            'title': title,
+            'description': clean_html(episode.get('description') or episode.get('summary')),
+            'thumbnail': episode.get('image'),
+            'timestamp': parse_iso8601(episode.get('publishDate')),
+            'duration': int_or_none(episode.get('duration')),
+            'filesize': int_or_none(episode.get('contentLength')),
+            'season_number': int_or_none(episode.get('season')),
+            'episode': title,
+            'episode_number': int_or_none(episode.get('episode')),
+        }
+        info.update(show_info)
+        return info
+
+    def _extract_show_info(self, show):
+        return {
+            'creator': show.get('author'),
+            'series': show.get('title'),
+        }
+
+    def _call_api(self, path, video_id, query=None):
+        return self._download_json(
+            'https://feeder.acast.com/api/v1/shows/' + path, video_id, query=query)
+
+
+class ACastIE(ACastBaseIE):
     IE_NAME = 'acast'
     _VALID_URL = r'''(?x)
                     https?://
@@ -28,15 +54,15 @@ class ACastIE(InfoExtractor):
                     '''
     _TESTS = [{
         'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna',
-        'md5': '16d936099ec5ca2d5869e3a813ee8dc4',
+        'md5': 'f5598f3ad1e4776fed12ec1407153e4b',
         'info_dict': {
             'id': '2a92b283-1a75-4ad8-8396-499c641de0d9',
             'ext': 'mp3',
             'title': '2. Raggarmordet - Röster ur det förflutna',
-            'description': 'md5:4f81f6d8cf2e12ee21a321d8bca32db4',
+            'description': 'md5:a992ae67f4d98f1c0141598f7bebbf67',
             'timestamp': 1477346700,
             'upload_date': '20161024',
-            'duration': 2766.602563,
+            'duration': 2766,
             'creator': 'Anton Berg & Martin Johnson',
             'series': 'Spår',
             'episode': '2. Raggarmordet - Röster ur det förflutna',
@@ -45,7 +71,7 @@ class ACastIE(InfoExtractor):
         'url': 'http://embed.acast.com/adambuxton/ep.12-adam-joeschristmaspodcast2015',
         'only_matching': True,
     }, {
-        'url': 'https://play.acast.com/s/rattegangspodden/s04e09-styckmordet-i-helenelund-del-22',
+        'url': 'https://play.acast.com/s/rattegangspodden/s04e09styckmordetihelenelund-del2-2',
         'only_matching': True,
     }, {
         'url': 'https://play.acast.com/s/sparpodcast/2a92b283-1a75-4ad8-8396-499c641de0d9',
@@ -54,40 +80,14 @@ class ACastIE(InfoExtractor):
 
     def _real_extract(self, url):
         channel, display_id = re.match(self._VALID_URL, url).groups()
-        s = self._download_json(
-            'https://feeder.acast.com/api/v1/shows/%s/episodes/%s' % (channel, display_id),
-            display_id)
-        media_url = s['url']
-        if re.search(r'[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}', display_id):
-            episode_url = s.get('episodeUrl')
-            if episode_url:
-                display_id = episode_url
-            else:
-                channel, display_id = re.match(self._VALID_URL, s['link']).groups()
-        cast_data = self._download_json(
-            'https://play-api.acast.com/splash/%s/%s' % (channel, display_id),
-            display_id)['result']
-        e = cast_data['episode']
-        title = e.get('name') or s['title']
-        return {
-            'id': compat_str(e['id']),
-            'display_id': display_id,
-            'url': media_url,
-            'title': title,
-            'description': e.get('summary') or clean_html(e.get('description') or s.get('description')),
-            'thumbnail': e.get('image'),
-            'timestamp': unified_timestamp(e.get('publishingDate') or s.get('publishDate')),
-            'duration': float_or_none(e.get('duration') or s.get('duration')),
-            'filesize': int_or_none(e.get('contentLength')),
-            'creator': try_get(cast_data, lambda x: x['show']['author'], compat_str),
-            'series': try_get(cast_data, lambda x: x['show']['name'], compat_str),
-            'season_number': int_or_none(e.get('seasonNumber')),
-            'episode': title,
-            'episode_number': int_or_none(e.get('episodeNumber')),
-        }
+        episode = self._call_api(
+            '%s/episodes/%s' % (channel, display_id),
+            display_id, {'showInfo': 'true'})
+        return self._extract_episode(
+            episode, self._extract_show_info(episode.get('show') or {}))
 
 
-class ACastChannelIE(InfoExtractor):
+class ACastChannelIE(ACastBaseIE):
     IE_NAME = 'acast:channel'
     _VALID_URL = r'''(?x)
                     https?://
@@ -102,34 +102,24 @@ class ACastChannelIE(InfoExtractor):
         'info_dict': {
             'id': '4efc5294-5385-4847-98bd-519799ce5786',
             'title': 'Today in Focus',
-            'description': 'md5:9ba5564de5ce897faeb12963f4537a64',
+            'description': 'md5:c09ce28c91002ce4ffce71d6504abaae',
         },
-        'playlist_mincount': 35,
+        'playlist_mincount': 200,
     }, {
         'url': 'http://play.acast.com/s/ft-banking-weekly',
         'only_matching': True,
     }]
-    _API_BASE_URL = 'https://play.acast.com/api/'
-    _PAGE_SIZE = 10
 
     @classmethod
     def suitable(cls, url):
         return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url)
 
-    def _fetch_page(self, channel_slug, page):
-        casts = self._download_json(
-            self._API_BASE_URL + 'channels/%s/acasts?page=%s' % (channel_slug, page),
-            channel_slug, note='Download page %d of channel data' % page)
-        for cast in casts:
-            yield self.url_result(
-                'https://play.acast.com/s/%s/%s' % (channel_slug, cast['url']),
-                'ACast', cast['id'])
-
     def _real_extract(self, url):
-        channel_slug = self._match_id(url)
-        channel_data = self._download_json(
-            self._API_BASE_URL + 'channels/%s' % channel_slug, channel_slug)
-        entries = OnDemandPagedList(functools.partial(
-            self._fetch_page, channel_slug), self._PAGE_SIZE)
-        return self.playlist_result(entries, compat_str(
-            channel_data['id']), channel_data['name'], channel_data.get('description'))
+        show_slug = self._match_id(url)
+        show = self._call_api(show_slug, show_slug)
+        show_info = self._extract_show_info(show)
+        entries = []
+        for episode in (show.get('episodes') or []):
+            entries.append(self._extract_episode(episode, show_info))
+        return self.playlist_result(
+            entries, show.get('id'), show.get('title'), show.get('description'))

youtube_dl/extractor/arcpublishing.py

@@ -0,0 +1,174 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    extract_attributes,
+    int_or_none,
+    parse_iso8601,
+    try_get,
+)
+
+
+class ArcPublishingIE(InfoExtractor):
+    _UUID_REGEX = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}'
+    _VALID_URL = r'arcpublishing:(?P<org>[a-z]+):(?P<id>%s)' % _UUID_REGEX
+    _TESTS = [{
+        # https://www.adn.com/politics/2020/11/02/video-senate-candidates-campaign-in-anchorage-on-eve-of-election-day/
+        'url': 'arcpublishing:adn:8c99cb6e-b29c-4bc9-9173-7bf9979225ab',
+        'only_matching': True,
+    }, {
+        # https://www.bostonglobe.com/video/2020/12/30/metro/footage-released-showing-officer-talking-about-striking-protesters-with-car/
+        'url': 'arcpublishing:bostonglobe:232b7ae6-7d73-432d-bc0a-85dbf0119ab1',
+        'only_matching': True,
+    }, {
+        # https://www.actionnewsjax.com/video/live-stream/
+        'url': 'arcpublishing:cmg:cfb1cf1b-3ab5-4d1b-86c5-a5515d311f2a',
+        'only_matching': True,
+    }, {
+        # https://elcomercio.pe/videos/deportes/deporte-total-futbol-peruano-seleccion-peruana-la-valorizacion-de-los-peruanos-en-el-exterior-tras-un-2020-atipico-nnav-vr-video-noticia/
+        'url': 'arcpublishing:elcomercio:27a7e1f8-2ec7-4177-874f-a4feed2885b3',
+        'only_matching': True,
+    }, {
+        # https://www.clickondetroit.com/video/community/2020/05/15/events-surrounding-woodward-dream-cruise-being-canceled/
+        'url': 'arcpublishing:gmg:c8793fb2-8d44-4242-881e-2db31da2d9fe',
+        'only_matching': True,
+    }, {
+        # https://www.wabi.tv/video/2020/12/30/trenton-company-making-equipment-pfizer-covid-vaccine/
+        'url': 'arcpublishing:gray:0b0ba30e-032a-4598-8810-901d70e6033e',
+        'only_matching': True,
+    }, {
+        # https://www.lateja.cr/el-mundo/video-china-aprueba-con-condiciones-su-primera/dfcbfa57-527f-45ff-a69b-35fe71054143/video/
+        'url': 'arcpublishing:gruponacion:dfcbfa57-527f-45ff-a69b-35fe71054143',
+        'only_matching': True,
+    }, {
+        # https://www.fifthdomain.com/video/2018/03/09/is-america-vulnerable-to-a-cyber-attack/
+        'url': 'arcpublishing:mco:aa0ca6fe-1127-46d4-b32c-be0d6fdb8055',
+        'only_matching': True,
+    }, {
+        # https://www.vl.no/kultur/2020/12/09/en-melding-fra-en-lytter-endret-julelista-til-lewi-bergrud/
+        'url': 'arcpublishing:mentormedier:47a12084-650b-4011-bfd0-3699b6947b2d',
+        'only_matching': True,
+    }, {
+        # https://www.14news.com/2020/12/30/whiskey-theft-caught-camera-henderson-liquor-store/
+        'url': 'arcpublishing:raycom:b89f61f8-79fa-4c09-8255-e64237119bf7',
+        'only_matching': True,
+    }, {
+        # https://www.theglobeandmail.com/world/video-ethiopian-woman-who-became-symbol-of-integration-in-italy-killed-on/
+        'url': 'arcpublishing:tgam:411b34c1-8701-4036-9831-26964711664b',
+        'only_matching': True,
+    }, {
+        # https://www.pilotonline.com/460f2931-8130-4719-8ea1-ffcb2d7cb685-132.html
+        'url': 'arcpublishing:tronc:460f2931-8130-4719-8ea1-ffcb2d7cb685',
+        'only_matching': True,
+    }]
+    _POWA_DEFAULTS = [
+        (['cmg', 'prisa'], '%s-config-prod.api.cdn.arcpublishing.com/video'),
+        ([
+            'adn', 'advancelocal', 'answers', 'bonnier', 'bostonglobe', 'demo',
+            'gmg', 'gruponacion', 'infobae', 'mco', 'nzme', 'pmn', 'raycom',
+            'spectator', 'tbt', 'tgam', 'tronc', 'wapo', 'wweek',
+        ], 'video-api-cdn.%s.arcpublishing.com/api'),
+    ]
+
+    @staticmethod
+    def _extract_urls(webpage):
+        entries = []
+        # https://arcpublishing.atlassian.net/wiki/spaces/POWA/overview
+        for powa_el in re.findall(r'(<div[^>]+class="[^"]*\bpowa\b[^"]*"[^>]+data-uuid="%s"[^>]*>)' % ArcPublishingIE._UUID_REGEX, webpage):
+            powa = extract_attributes(powa_el) or {}
+            org = powa.get('data-org')
+            uuid = powa.get('data-uuid')
+            if org and uuid:
+                entries.append('arcpublishing:%s:%s' % (org, uuid))
+        return entries
+
+    def _real_extract(self, url):
+        org, uuid = re.match(self._VALID_URL, url).groups()
+        for orgs, tmpl in self._POWA_DEFAULTS:
+            if org in orgs:
+                base_api_tmpl = tmpl
+                break
+        else:
+            base_api_tmpl = '%s-prod-cdn.video-api.arcpublishing.com/api'
+        if org == 'wapo':
+            org = 'washpost'
+        video = self._download_json(
+            'https://%s/v1/ansvideos/findByUuid' % (base_api_tmpl % org),
+            uuid, query={'uuid': uuid})[0]
+        title = video['headlines']['basic']
+        is_live = video.get('status') == 'live'
+
+        urls = []
+        formats = []
+        for s in video.get('streams', []):
+            s_url = s.get('url')
+            if not s_url or s_url in urls:
+                continue
+            urls.append(s_url)
+            stream_type = s.get('stream_type')
+            if stream_type == 'smil':
+                smil_formats = self._extract_smil_formats(
+                    s_url, uuid, fatal=False)
+                for f in smil_formats:
+                    if f['url'].endswith('/cfx/st'):
+                        f['app'] = 'cfx/st'
+                        if not f['play_path'].startswith('mp4:'):
+                            f['play_path'] = 'mp4:' + f['play_path']
+                        if isinstance(f['tbr'], float):
+                            f['vbr'] = f['tbr'] * 1000
+                            del f['tbr']
+                            f['format_id'] = 'rtmp-%d' % f['vbr']
+                formats.extend(smil_formats)
+            elif stream_type in ('ts', 'hls'):
+                m3u8_formats = self._extract_m3u8_formats(
+                    s_url, uuid, 'mp4', 'm3u8' if is_live else 'm3u8_native',
+                    m3u8_id='hls', fatal=False)
+                if all([f.get('acodec') == 'none' for f in m3u8_formats]):
+                    continue
+                for f in m3u8_formats:
+                    if f.get('acodec') == 'none':
+                        f['preference'] = -40
+                    elif f.get('vcodec') == 'none':
+                        f['preference'] = -50
+                    height = f.get('height')
+                    if not height:
+                        continue
+                    vbr = self._search_regex(
+                        r'[_x]%d[_-](\d+)' % height, f['url'], 'vbr', default=None)
+                    if vbr:
+                        f['vbr'] = int(vbr)
+                formats.extend(m3u8_formats)
+            else:
+                vbr = int_or_none(s.get('bitrate'))
+                formats.append({
+                    'format_id': '%s-%d' % (stream_type, vbr) if vbr else stream_type,
+                    'vbr': vbr,
+                    'width': int_or_none(s.get('width')),
+                    'height': int_or_none(s.get('height')),
+                    'filesize': int_or_none(s.get('filesize')),
+                    'url': s_url,
+                    'preference': -1,
+                })
+        self._sort_formats(
+            formats, ('preference', 'width', 'height', 'vbr', 'filesize', 'tbr', 'ext', 'format_id'))
+
+        subtitles = {}
+        for subtitle in (try_get(video, lambda x: x['subtitles']['urls'], list) or []):
+            subtitle_url = subtitle.get('url')
+            if subtitle_url:
+                subtitles.setdefault('en', []).append({'url': subtitle_url})
+
+        return {
+            'id': uuid,
+            'title': self._live_title(title) if is_live else title,
+            'thumbnail': try_get(video, lambda x: x['promo_image']['url']),
+            'description': try_get(video, lambda x: x['subheadlines']['basic']),
+            'formats': formats,
+            'duration': int_or_none(video.get('duration'), 100),
+            'timestamp': parse_iso8601(video.get('created_date')),
+            'subtitles': subtitles,
+            'is_live': is_live,
+        }

youtube_dl/extractor/common.py

@@ -2135,17 +2135,15 @@ def extract_common(source):
             def extract_Initialization(source):
                 initialization = source.find(_add_ns('Initialization'))
                 if initialization is not None:
-                    ms_info['initialization_url'] = initialization.get('sourceURL')
+                    ms_info['initialization_url'] = initialization.attrib['sourceURL']
 
             segment_list = element.find(_add_ns('SegmentList'))
             if segment_list is not None:
                 extract_common(segment_list)
                 extract_Initialization(segment_list)
                 segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
                 if segment_urls_e:
-                    segment_urls = [segment.get('media') for segment in segment_urls_e if segment.get('media') is not None]
-                    if len(segment_urls) > 0:
-                        ms_info['segment_urls'] = segment_urls
+                    ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
             else:
                 segment_template = element.find(_add_ns('SegmentTemplate'))
                 if segment_template is not None:

youtube_dl/extractor/extractors.py

@@ -56,6 +56,7 @@
     AppleTrailersSectionIE,
 )
 from .archiveorg import ArchiveOrgIE
+from .arcpublishing import ArcPublishingIE
 from .arkena import ArkenaIE
 from .ard import (
     ARDBetaMediathekIE,
@@ -789,6 +790,7 @@
     NRKSkoleIE,
     NRKTVIE,
     NRKTVDirekteIE,
+    NRKRadioPodkastIE,
     NRKTVEpisodeIE,
     NRKTVEpisodesIE,
     NRKTVSeasonIE,
@@ -1057,6 +1059,7 @@
 from .sky import (
     SkyNewsIE,
     SkySportsIE,
+    SkySportsNewsIE,
 )
 from .slideshare import SlideshareIE
 from .slideslive import SlidesLiveIE