commit youtube-dl for openSUSE:Factory
Hello community, here is the log from the commit of package youtube-dl for openSUSE:Factory checked in at 2018-04-30 22:57:20 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/youtube-dl (Old) and /work/SRC/openSUSE:Factory/.youtube-dl.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "youtube-dl" Mon Apr 30 22:57:20 2018 rev:74 rq:602414 version:2018.04.25 Changes: -------- --- /work/SRC/openSUSE:Factory/youtube-dl/python-youtube-dl.changes 2018-04-17 11:18:15.780314514 +0200 +++ /work/SRC/openSUSE:Factory/.youtube-dl.new/python-youtube-dl.changes 2018-04-30 22:59:19.239624832 +0200 @@ -1,0 +2,27 @@ +Wed Apr 25 07:48:22 UTC 2018 - seb95.scou@gmail.com + +- Update to new upstream release 2018.04.25 + * Fix match_str for boolean meta fields + * Add support for pandoc 2 and disable smart extension (#16251) + * Fix typo in media extension compatibility checker (#16215) + * Recognize IPv6 stream URLs (#16136, #16137, #16205, #16246, + #16250) + * Extract is_live according to status (#16259) + * Relax URL regular expression (#16258) + * Remove extractor (#16256) + * Fix extraction (#16254) + * Add ability to authenticate with cookies + * Implement lazy playlist extraction (#10184) + * Add support for TV channel live streams (#15279, #15809) + * Fix video extraction (#15931) + * Fix extraction (#15227) + * Add support for nickjr.nl (#16230) + * Fix metadata extraction + * Add support for generic embeds (#16134, #16154) + * Extract new azure URLs (#16223) + * Fix extraction (#16217) + * Improve embeds detection (#16201) + * Fix extraction (#16119) + * Skip DRM asset types (#16104) + +------------------------------------------------------------------- youtube-dl.changes: same change Old: ---- youtube-dl-2018.04.16.tar.gz youtube-dl-2018.04.16.tar.gz.sig New: ---- youtube-dl-2018.04.25.tar.gz youtube-dl-2018.04.25.tar.gz.sig ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-youtube-dl.spec ++++++ --- /var/tmp/diff_new_pack.NFRUak/_old 2018-04-30 22:59:20.795568056 +0200 +++ /var/tmp/diff_new_pack.NFRUak/_new 2018-04-30 22:59:20.803567764 +0200 @@ -19,7 +19,7 @@ %define modname youtube-dl %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-youtube-dl -Version: 2018.04.16 +Version: 2018.04.25 Release: 0 Summary: A python module for downloading from video sites for offline watching License: SUSE-Public-Domain AND CC-BY-SA-3.0 ++++++ youtube-dl.spec ++++++ --- /var/tmp/diff_new_pack.NFRUak/_old 2018-04-30 22:59:20.823567035 +0200 +++ /var/tmp/diff_new_pack.NFRUak/_new 2018-04-30 22:59:20.827566889 +0200 @@ -17,7 +17,7 @@ Name: youtube-dl -Version: 2018.04.16 +Version: 2018.04.25 Release: 0 Summary: A tool for downloading from video sites for offline watching License: SUSE-Public-Domain AND CC-BY-SA-3.0 ++++++ youtube-dl-2018.04.16.tar.gz -> youtube-dl-2018.04.25.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/AUTHORS new/youtube-dl/AUTHORS --- old/youtube-dl/AUTHORS 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/AUTHORS 2018-04-24 20:11:43.000000000 +0200 @@ -236,3 +236,6 @@ Petr Novák Leonardo Taccari Martin Weinelt +Surya Oktafendri +TingPing +Alexandre Macabies diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/ChangeLog new/youtube-dl/ChangeLog --- old/youtube-dl/ChangeLog 2018-04-15 20:09:12.000000000 +0200 +++ new/youtube-dl/ChangeLog 2018-04-24 20:12:36.000000000 +0200 @@ -1,3 +1,32 @@ +version 2018.04.25 + +Core +* [utils] Fix match_str for boolean meta fields ++ [Makefile] Add support for pandoc 2 and disable smart extension (#16251) +* [YoutubeDL] Fix typo in media extension compatibility checker (#16215) + +Extractors ++ [openload] Recognize IPv6 stream URLs (#16136, #16137, #16205, #16246, + #16250) ++ [twitch] Extract is_live according to status (#16259) +* [pornflip] Relax URL regular expression (#16258) +- [etonline] Remove extractor (#16256) +* [breakcom] Fix extraction (#16254) ++ [youtube] Add ability to authenticate with cookies +* [youtube:feed] Implement lazy playlist extraction (#10184) ++ [svt] Add support for TV channel live streams (#15279, #15809) +* [ccma] Fix video extraction (#15931) +* [rentv] Fix extraction (#15227) ++ [nick] Add support for nickjr.nl (#16230) +* [extremetube] Fix metadata extraction ++ [keezmovies] Add support for generic embeds (#16134, #16154) +* [nexx] Extract new azure URLs (#16223) +* [cbssports] Fix extraction (#16217) +* [kaltura] Improve embeds detection (#16201) +* [instagram:user] Fix extraction (#16119) +* [cbs] Skip DRM asset types (#16104) + + version 2018.04.16 Extractors diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/Makefile new/youtube-dl/Makefile --- old/youtube-dl/Makefile 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/Makefile 2018-04-24 20:11:43.000000000 +0200 @@ -14,6 +14,9 @@ # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then echo /etc; else echo $(PREFIX)/etc; fi) +# set markdown input format to "markdown-smart" for pandoc version 2 and to "markdown" for pandoc prior to version 2 +MARKDOWN = $(shell if [ `pandoc -v | head -n1 | cut -d" " -f2 | head -c1` = "2" ]; then echo markdown-smart; else echo markdown; fi) + install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish install -d $(DESTDIR)$(BINDIR) install -m 755 youtube-dl $(DESTDIR)$(BINDIR) @@ -82,11 +85,11 @@ $(PYTHON) devscripts/make_supportedsites.py docs/supportedsites.md README.txt: README.md - pandoc -f markdown -t plain README.md -o README.txt + pandoc -f $(MARKDOWN) -t plain README.md -o README.txt youtube-dl.1: README.md $(PYTHON) devscripts/prepare_manpage.py youtube-dl.1.temp.md - pandoc -s -f markdown -t man youtube-dl.1.temp.md -o youtube-dl.1 + pandoc -s -f $(MARKDOWN) -t man youtube-dl.1.temp.md -o youtube-dl.1 rm -f youtube-dl.1.temp.md youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-completion.in diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/README.txt new/youtube-dl/README.txt --- old/youtube-dl/README.txt 2018-04-15 20:09:46.000000000 +0200 +++ new/youtube-dl/README.txt 2018-04-24 20:12:56.000000000 +0200 @@ -1353,12 +1353,12 @@ 1. Fork this repository 2. Check out the source code with: - git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git + git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git 3. Start a new git branch with - cd youtube-dl - git checkout -b yourextractor + cd youtube-dl + git checkout -b yourextractor 4. Start with this simple template and save it to youtube_dl/extractor/yourextractor.py: @@ -1421,10 +1421,10 @@ 9. When the tests pass, add the new files and commit them and push the result, like this: - $ git add youtube_dl/extractor/extractors.py - $ git add youtube_dl/extractor/yourextractor.py - $ git commit -m '[yourextractor] Add new extractor' - $ git push origin yourextractor + $ git add youtube_dl/extractor/extractors.py + $ git add youtube_dl/extractor/yourextractor.py + $ git commit -m '[yourextractor] Add new extractor' + $ git push origin yourextractor 10. Finally, create a pull request. We'll then review and merge it. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/docs/supportedsites.md new/youtube-dl/docs/supportedsites.md --- old/youtube-dl/docs/supportedsites.md 2018-04-15 20:09:18.000000000 +0200 +++ new/youtube-dl/docs/supportedsites.md 2018-04-24 20:12:40.000000000 +0200 @@ -257,7 +257,6 @@ - **ESPN** - **ESPNArticle** - **EsriVideo** - - **ETOnline** - **Europa** - **EveryonesMixtape** - **ExpoTV** diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/test/test_subtitles.py new/youtube-dl/test/test_subtitles.py --- old/youtube-dl/test/test_subtitles.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/test/test_subtitles.py 2018-04-24 20:11:37.000000000 +0200 @@ -232,7 +232,7 @@ class TestMTVSubtitles(BaseTestSubtitles): - url = 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored--...' + url = 'http://www.cc.com/video-clips/p63lk0/adam-devine-s-house-party-chasing-white...' IE = ComedyCentralIE def getInfoDict(self): @@ -243,7 +243,7 @@ self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() self.assertEqual(set(subtitles.keys()), set(['en'])) - self.assertEqual(md5(subtitles['en']), 'b9f6ca22a6acf597ec76f61749765e65') + self.assertEqual(md5(subtitles['en']), '78206b8d8a0cfa9da64dc026eea48961') class TestNRKSubtitles(BaseTestSubtitles): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/test/test_utils.py new/youtube-dl/test/test_utils.py --- old/youtube-dl/test/test_utils.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/test/test_utils.py 2018-04-24 20:11:43.000000000 +0200 @@ -1072,6 +1072,18 @@ self.assertFalse(match_str( 'like_count > 100 & dislike_count <? 50 & description', {'like_count': 190, 'dislike_count': 10})) + self.assertTrue(match_str('is_live', {'is_live': True})) + self.assertFalse(match_str('is_live', {'is_live': False})) + self.assertFalse(match_str('is_live', {'is_live': None})) + self.assertFalse(match_str('is_live', {})) + self.assertFalse(match_str('!is_live', {'is_live': True})) + self.assertTrue(match_str('!is_live', {'is_live': False})) + self.assertTrue(match_str('!is_live', {'is_live': None})) + self.assertTrue(match_str('!is_live', {})) + self.assertTrue(match_str('title', {'title': 'abc'})) + self.assertTrue(match_str('title', {'title': ''})) + self.assertFalse(match_str('!title', {'title': 'abc'})) + self.assertFalse(match_str('!title', {'title': ''})) def test_parse_dfxp_time_expr(self): self.assertEqual(parse_dfxp_time_expr(None), None) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/test/test_youtube_lists.py new/youtube-dl/test/test_youtube_lists.py --- old/youtube-dl/test/test_youtube_lists.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/test/test_youtube_lists.py 2018-04-24 20:11:37.000000000 +0200 @@ -61,7 +61,7 @@ dl = FakeYDL() dl.params['extract_flat'] = True ie = YoutubePlaylistIE(dl) - result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re') + result = ie.extract('https://www.youtube.com/playlist?list=PL-KKIb8rvtMSrAO9YFbeM6UQrAqoFTUWv') self.assertIsPlaylist(result) for entry in result['entries']: self.assertTrue(entry.get('title')) Binary files old/youtube-dl/youtube-dl and new/youtube-dl/youtube-dl differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube-dl.1 new/youtube-dl/youtube-dl.1 --- old/youtube-dl/youtube-dl.1 2018-04-15 20:09:47.000000000 +0200 +++ new/youtube-dl/youtube-dl.1 2018-04-24 20:12:57.000000000 +0200 @@ -1,4 +1,7 @@ +.\" Automatically generated by Pandoc 2.1.3 +.\" .TH "YOUTUBE\-DL" "1" "" "" "" +.hy .SH NAME .PP youtube\-dl \- download videos from youtube.com or other video platforms @@ -997,7 +1000,7 @@ .PP To activate authentication with the \f[C]\&.netrc\f[] file you should pass \f[C]\-\-netrc\f[] to youtube\-dl or place it in the configuration -file (#configuration). +file. .PP On Windows you may also need to setup the \f[C]%HOME%\f[] environment variable manually. @@ -1013,7 +1016,7 @@ The \f[C]\-o\f[] option allows users to indicate a template for the output file names. .PP -\f[B]tl;dr:\f[] navigate me to examples (#output-template-examples). +\f[B]tl;dr:\f[] navigate me to examples. .PP The basic usage is not to set any template arguments when downloading a single file, like in @@ -1293,7 +1296,7 @@ an expression that describes format or formats you would like to download. .PP -\f[B]tl;dr:\f[] navigate me to examples (#format-selection-examples). +\f[B]tl;dr:\f[] navigate me to examples. .PP The simplest case is requesting a specific format, for example with \f[C]\-f\ 22\f[] you can download the format with format code equal to @@ -1418,8 +1421,8 @@ youtube\-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with \f[C]\-f\ best\f[]. -You may want to add it to the configuration file (#configuration) in -order not to type it every time you run youtube\-dl. +You may want to add it to the configuration file in order not to type it +every time you run youtube\-dl. .SS Format selection examples .PP Note that on Windows you may need to use double quotes instead of @@ -1629,12 +1632,12 @@ .PP YouTube has switched to a new video info format in July 2011 which is not supported by old versions of youtube\-dl. -See above (#how-do-i-update-youtube-dl) for how to update youtube\-dl. +See above for how to update youtube\-dl. .SS ERROR: unable to download video .PP YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube\-dl. -See above (#how-do-i-update-youtube-dl) for how to update youtube\-dl. +See above for how to update youtube\-dl. .SS Video URL contains an ampersand and I\[aq]m getting some strange output \f[C][1]\ 2839\f[] or \f[C]\[aq]v\[aq]\ is\ not\ recognized\ as\ an\ internal\ or\ external\ command\f[] @@ -1665,15 +1668,15 @@ .PP In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube\-dl. -See above (#how-do-i-update-youtube-dl) for how to update youtube\-dl. +See above for how to update youtube\-dl. .SS HTTP Error 429: Too Many Requests or 402: Payment Required .PP These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or \- if you have acquired a whitelisted IP address already \- use the -\f[C]\-\-proxy\f[] or \f[C]\-\-source\-address\f[] -options (#network-options) to select another IP address. +\f[C]\-\-proxy\f[] or \f[C]\-\-source\-address\f[] options to select +another IP address. .SS SyntaxError: Non\-ASCII character .PP The error @@ -1721,10 +1724,10 @@ matter what directory you\[aq]re in. .SS How do I put downloads into a specific folder? .PP -Use the \f[C]\-o\f[] to specify an output template (#output-template), -for example \f[C]\-o\ "/home/user/videos/%(title)s\-%(id)s.%(ext)s"\f[]. +Use the \f[C]\-o\f[] to specify an output template, for example +\f[C]\-o\ "/home/user/videos/%(title)s\-%(id)s.%(ext)s"\f[]. If you want this for all of your downloads, put the option into your -configuration file (#configuration). +configuration file. .SS How do I download a video starting with a \f[C]\-\f[]? .PP Either prepend \f[C]https://www.youtube.com/watch?v=\f[] or separate the @@ -1872,7 +1875,7 @@ Unfortunately, the youtube\-dl project has grown too large to use personal email as an effective communication channel. .PP -Please read the bug reporting instructions (#bugs) below. +Please read the bug reporting instructions below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the youtube\-dl developers. @@ -1918,15 +1921,14 @@ .SH Why do I need to go through that much red tape when filing bugs? .PP Before we had the issue template, despite our extensive bug reporting -instructions (#bugs), about 80% of the issue reports we got were -useless, for instance because people used ancient versions hundreds of -releases old, because of simple syntactic errors (not in youtube\-dl but -in general shell usage), because the problem was already reported -multiple times before, because people did not actually read an error -message, even if it said "please install ffmpeg", because people did not -mention the URL they were trying to download and many more simple, -easy\-to\-avoid problems, many of whom were totally unrelated to -youtube\-dl. +instructions, about 80% of the issue reports we got were useless, for +instance because people used ancient versions hundreds of releases old, +because of simple syntactic errors (not in youtube\-dl but in general +shell usage), because the problem was already reported multiple times +before, because people did not actually read an error message, even if +it said "please install ffmpeg", because people did not mention the URL +they were trying to download and many more simple, easy\-to\-avoid +problems, many of whom were totally unrelated to youtube\-dl. .PP youtube\-dl is an open\-source project manned by too few volunteers, so we\[aq]d rather spend time fixing bugs where we are certain none of @@ -1963,8 +1965,8 @@ \f[] .fi .PP -See item 6 of new extractor tutorial (#adding-support-for-a-new-site) -for how to run extractor specific test cases. +See item 6 of new extractor tutorial for how to run extractor specific +test cases. .PP If you want to create a build of youtube\-dl yourself, you\[aq]ll need .IP \[bu] 2 @@ -1996,7 +1998,7 @@ .IP .nf \f[C] -git\ clone\ git\@github.com:YOUR_GITHUB_USERNAME/youtube\-dl.git +\ git\ clone\ git\@github.com:YOUR_GITHUB_USERNAME/youtube\-dl.git \f[] .fi .RE @@ -2006,8 +2008,8 @@ .IP .nf \f[C] -cd\ youtube\-dl -git\ checkout\ \-b\ yourextractor +\ cd\ youtube\-dl +\ git\ checkout\ \-b\ yourextractor \f[] .fi .RE @@ -2082,9 +2084,8 @@ return (https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py...). Add tests and code for as many as you want. .IP " 8." 4 -Make sure your code follows youtube\-dl coding -conventions (#youtube-dl-coding-conventions) and check the code with -flake8 (https://pypi.python.org/pypi/flake8). +Make sure your code follows youtube\-dl coding conventions and check the +code with flake8 (https://pypi.python.org/pypi/flake8). Also make sure your code works under all Python (https://www.python.org/) versions claimed supported by youtube\-dl, namely 2.6, 2.7, and 3.2+. @@ -2096,10 +2097,10 @@ .IP .nf \f[C] -$\ git\ add\ youtube_dl/extractor/extractors.py -$\ git\ add\ youtube_dl/extractor/yourextractor.py -$\ git\ commit\ \-m\ \[aq][yourextractor]\ Add\ new\ extractor\[aq] -$\ git\ push\ origin\ yourextractor +\ $\ git\ add\ youtube_dl/extractor/extractors.py +\ $\ git\ add\ youtube_dl/extractor/yourextractor.py +\ $\ git\ commit\ \-m\ \[aq][yourextractor]\ Add\ new\ extractor\[aq] +\ $\ git\ push\ origin\ yourextractor \f[] .fi .RE @@ -2404,7 +2405,7 @@ (webchat (https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)). .PP \f[B]Please include the full output of youtube\-dl when run with -\f[C]\-v\f[]\f[], i.e. +\f[BC]\-v\f[B]\f[], i.e. \f[B]add\f[] \f[C]\-v\f[] flag to \f[B]your command line\f[], copy the \f[B]whole\f[] output and post it in the issue body wrapped in ``` for better formatting. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/YoutubeDL.py new/youtube-dl/youtube_dl/YoutubeDL.py --- old/youtube-dl/youtube_dl/YoutubeDL.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/YoutubeDL.py 2018-04-24 20:11:37.000000000 +0200 @@ -1853,7 +1853,7 @@ def compatible_formats(formats): video, audio = formats # Check extension - video_ext, audio_ext = audio.get('ext'), video.get('ext') + video_ext, audio_ext = video.get('ext'), audio.get('ext') if video_ext and audio_ext: COMPATIBLE_EXTS = ( ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'), diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/breakcom.py new/youtube-dl/youtube_dl/extractor/breakcom.py --- old/youtube-dl/youtube_dl/extractor/breakcom.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/breakcom.py 2018-04-24 20:11:37.000000000 +0200 @@ -3,15 +3,13 @@ import re from .common import InfoExtractor +from .youtube import YoutubeIE from ..compat import compat_str -from ..utils import ( - int_or_none, - parse_age_limit, -) +from ..utils import int_or_none class BreakIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?(?P<site>break|screenjunkies)\.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)' + _VALID_URL = r'https?://(?:www\.)?break\.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)' _TESTS = [{ 'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056', 'info_dict': { @@ -19,125 +17,73 @@ 'ext': 'mp4', 'title': 'When Girls Act Like D-Bags', 'age_limit': 13, - } - }, { - 'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915', - 'md5': '5c2b686bec3d43de42bde9ec047536b0', - 'info_dict': { - 'id': '2841915', - 'display_id': 'best-quentin-tarantino-movie', - 'ext': 'mp4', - 'title': 'Best Quentin Tarantino Movie', - 'thumbnail': r're:^https?://.*\.jpg', - 'duration': 3671, - 'age_limit': 13, - 'tags': list, }, }, { - 'url': 'http://www.screenjunkies.com/video/honest-trailers-the-dark-knight', + # youtube embed + 'url': 'http://www.break.com/video/someone-forgot-boat-brakes-work', 'info_dict': { - 'id': '2348808', - 'display_id': 'honest-trailers-the-dark-knight', + 'id': 'RrrDLdeL2HQ', 'ext': 'mp4', - 'title': 'Honest Trailers - The Dark Knight', - 'thumbnail': r're:^https?://.*\.(?:jpg|png)', - 'age_limit': 10, - 'tags': list, - }, - }, { - # requires subscription but worked around - 'url': 'http://www.screenjunkies.com/video/knocking-dead-ep-1-the-show-so-far-300328...', - 'info_dict': { - 'id': '3003285', - 'display_id': 'knocking-dead-ep-1-the-show-so-far', - 'ext': 'mp4', - 'title': 'State of The Dead Recap: Knocking Dead Pilot', - 'thumbnail': r're:^https?://.*\.jpg', - 'duration': 3307, - 'age_limit': 13, - 'tags': list, + 'title': 'Whale Watching Boat Crashing Into San Diego Dock', + 'description': 'md5:afc1b2772f0a8468be51dd80eb021069', + 'upload_date': '20160331', + 'uploader': 'Steve Holden', + 'uploader_id': 'sdholden07', }, + 'params': { + 'skip_download': True, + } }, { 'url': 'http://www.break.com/video/ugc/baby-flex-2773063', 'only_matching': True, }] - _DEFAULT_BITRATES = (48, 150, 320, 496, 864, 2240, 3264) - def _real_extract(self, url): - site, display_id, video_id = re.match(self._VALID_URL, url).groups() + display_id, video_id = re.match(self._VALID_URL, url).groups() - if not video_id: - webpage = self._download_webpage(url, display_id) - video_id = self._search_regex( - (r'src=["\']/embed/(\d+)', r'data-video-content-id=["\'](\d+)'), - webpage, 'video id') - - webpage = self._download_webpage( - 'http://www.%s.com/embed/%s' % (site, video_id), - display_id, 'Downloading video embed page') - embed_vars = self._parse_json( - self._search_regex( - r'(?s)embedVars\s*=\s*({.+?})\s*</script>', webpage, 'embed vars'), - display_id) + webpage = self._download_webpage(url, display_id) - youtube_id = embed_vars.get('youtubeId') - if youtube_id: - return self.url_result(youtube_id, 'Youtube') + youtube_url = YoutubeIE._extract_url(webpage) + if youtube_url: + return self.url_result(youtube_url, ie=YoutubeIE.ie_key()) - title = embed_vars['contentName'] + content = self._parse_json( + self._search_regex( + r'(?s)content["\']\s*:\s*(\[.+?\])\s*[,\n]', webpage, + 'content'), + display_id) formats = [] - bitrates = [] - for f in embed_vars.get('media', []): - if not f.get('uri') or f.get('mediaPurpose') != 'play': + for video in content: + video_url = video.get('url') + if not video_url or not isinstance(video_url, compat_str): continue - bitrate = int_or_none(f.get('bitRate')) - if bitrate: - bitrates.append(bitrate) + bitrate = int_or_none(self._search_regex( + r'(\d+)_kbps', video_url, 'tbr', default=None)) formats.append({ - 'url': f['uri'], + 'url': video_url, 'format_id': 'http-%d' % bitrate if bitrate else 'http', - 'width': int_or_none(f.get('width')), - 'height': int_or_none(f.get('height')), 'tbr': bitrate, - 'format': 'mp4', }) - - if not bitrates: - # When subscriptionLevel > 0, i.e. plus subscription is required - # media list will be empty. However, hds and hls uris are still - # available. We can grab them assuming bitrates to be default. - bitrates = self._DEFAULT_BITRATES - - auth_token = embed_vars.get('AuthToken') - - def construct_manifest_url(base_url, ext): - pieces = [base_url] - pieces.extend([compat_str(b) for b in bitrates]) - pieces.append('_kbps.mp4.%s?%s' % (ext, auth_token)) - return ','.join(pieces) - - if bitrates and auth_token: - hds_url = embed_vars.get('hdsUri') - if hds_url: - formats.extend(self._extract_f4m_formats( - construct_manifest_url(hds_url, 'f4m'), - display_id, f4m_id='hds', fatal=False)) - hls_url = embed_vars.get('hlsUri') - if hls_url: - formats.extend(self._extract_m3u8_formats( - construct_manifest_url(hls_url, 'm3u8'), - display_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)) self._sort_formats(formats) + title = self._search_regex( + (r'title["\']\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1', + r'<h1[^>]*>(?P<value>[^<]+)'), webpage, 'title', group='value') + + def get(key, name): + return int_or_none(self._search_regex( + r'%s["\']\s*:\s*["\'](\d+)' % key, webpage, name, + default=None)) + + age_limit = get('ratings', 'age limit') + video_id = video_id or get('pid', 'video id') or display_id + return { 'id': video_id, 'display_id': display_id, 'title': title, - 'thumbnail': embed_vars.get('thumbUri'), - 'duration': int_or_none(embed_vars.get('videoLengthInSeconds')) or None, - 'age_limit': parse_age_limit(embed_vars.get('audienceRating')), - 'tags': embed_vars.get('tags', '').split(','), + 'thumbnail': self._og_search_thumbnail(webpage), + 'age_limit': age_limit, 'formats': formats, } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/cbs.py new/youtube-dl/youtube_dl/extractor/cbs.py --- old/youtube-dl/youtube_dl/extractor/cbs.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/cbs.py 2018-04-24 20:11:37.000000000 +0200 @@ -65,7 +65,7 @@ last_e = None for item in items_data.findall('.//item'): asset_type = xpath_text(item, 'assetType') - if not asset_type or asset_type in asset_types: + if not asset_type or asset_type in asset_types or asset_type in ('HLS_FPS', 'DASH_CENC'): continue asset_types.append(asset_type) query = { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/cbssports.py new/youtube-dl/youtube_dl/extractor/cbssports.py --- old/youtube-dl/youtube_dl/extractor/cbssports.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/cbssports.py 2018-04-24 20:11:37.000000000 +0200 @@ -4,28 +4,35 @@ class CBSSportsIE(CBSBaseIE): - _VALID_URL = r'https?://(?:www\.)?cbssports\.com/video/player/[^/]+/(?P<id>\d+)' + _VALID_URL = r'https?://(?:www\.)?cbssports\.com/[^/]+/(?:video|news)/(?P<id>[^/?#&]+)' _TESTS = [{ - 'url': 'http://www.cbssports.com/video/player/videos/708337219968/0/ben-simmons-the-...', + 'url': 'https://www.cbssports.com/nba/video/donovan-mitchell-flashes-star-potential-...', 'info_dict': { - 'id': '708337219968', + 'id': '1214315075735', 'ext': 'mp4', - 'title': 'Ben Simmons the next LeBron? Not so fast', - 'description': 'md5:854294f627921baba1f4b9a990d87197', - 'timestamp': 1466293740, - 'upload_date': '20160618', + 'title': 'Donovan Mitchell flashes star potential in Game 2 victory over Thunder', + 'description': 'md5:df6f48622612c2d6bd2e295ddef58def', + 'timestamp': 1524111457, + 'upload_date': '20180419', 'uploader': 'CBSI-NEW', }, 'params': { # m3u8 download 'skip_download': True, } + }, { + 'url': 'https://www.cbssports.com/nba/news/nba-playoffs-2018-watch-76ers-vs-heat-gam...', + 'only_matching': True, }] def _extract_video_info(self, filter_query, video_id): return self._extract_feed_info('dJ5BDC', 'VxxJg8Ymh8sE', filter_query, video_id) def _real_extract(self, url): - video_id = self._match_id(url) + display_id = self._match_id(url) + webpage = self._download_webpage(url, display_id) + video_id = self._search_regex( + [r'(?:=|%26)pcid%3D(\d+)', r'embedVideo(?:Container)?_(\d+)'], + webpage, 'video id') return self._extract_video_info('byId=%s' % video_id, video_id) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/ccma.py new/youtube-dl/youtube_dl/extractor/ccma.py --- old/youtube-dl/youtube_dl/extractor/ccma.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/ccma.py 2018-04-24 20:11:37.000000000 +0200 @@ -4,11 +4,13 @@ import re from .common import InfoExtractor +from ..compat import compat_str from ..utils import ( + clean_html, int_or_none, parse_duration, parse_iso8601, - clean_html, + parse_resolution, ) @@ -40,34 +42,42 @@ def _real_extract(self, url): media_type, media_id = re.match(self._VALID_URL, url).groups() - media_data = {} - formats = [] - profiles = ['pc'] if media_type == 'audio' else ['mobil', 'pc'] - for i, profile in enumerate(profiles): - md = self._download_json('http://dinamics.ccma.cat/pvideo/media.jsp', media_id, query={ + + media = self._download_json( + 'http://dinamics.ccma.cat/pvideo/media.jsp', media_id, query={ 'media': media_type, 'idint': media_id, - 'profile': profile, - }, fatal=False) - if md: - media_data = md - media_url = media_data.get('media', {}).get('url') - if media_url: - formats.append({ - 'format_id': profile, - 'url': media_url, - 'quality': i, - }) + }) + + formats = [] + media_url = media['media']['url'] + if isinstance(media_url, list): + for format_ in media_url: + format_url = format_.get('file') + if not format_url or not isinstance(format_url, compat_str): + continue + label = format_.get('label') + f = parse_resolution(label) + f.update({ + 'url': format_url, + 'format_id': label, + }) + formats.append(f) + else: + formats.append({ + 'url': media_url, + 'vcodec': 'none' if media_type == 'audio' else None, + }) self._sort_formats(formats) - informacio = media_data['informacio'] + informacio = media['informacio'] title = informacio['titol'] durada = informacio.get('durada', {}) duration = int_or_none(durada.get('milisegons'), 1000) or parse_duration(durada.get('text')) timestamp = parse_iso8601(informacio.get('data_emissio', {}).get('utc')) subtitles = {} - subtitols = media_data.get('subtitols', {}) + subtitols = media.get('subtitols', {}) if subtitols: sub_url = subtitols.get('url') if sub_url: @@ -77,7 +87,7 @@ }) thumbnails = [] - imatges = media_data.get('imatges', {}) + imatges = media.get('imatges', {}) if imatges: thumbnail_url = imatges.get('url') if thumbnail_url: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/etonline.py new/youtube-dl/youtube_dl/extractor/etonline.py --- old/youtube-dl/youtube_dl/extractor/etonline.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/etonline.py 1970-01-01 01:00:00.000000000 +0100 @@ -1,39 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor - - -class ETOnlineIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?etonline\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)' - _TESTS = [{ - 'url': 'http://www.etonline.com/tv/211130_dove_cameron_liv_and_maddie_emotional_epis...', - 'info_dict': { - 'id': '211130_dove_cameron_liv_and_maddie_emotional_episode_series_finale', - 'title': 'md5:a21ec7d3872ed98335cbd2a046f34ee6', - 'description': 'md5:8b94484063f463cca709617c79618ccd', - }, - 'playlist_count': 2, - }, { - 'url': 'http://www.etonline.com/media/video/here_are_the_stars_who_love_bringing_the...', - 'only_matching': True, - }] - BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1242911076001/default_default/index.html?video...' - - def _real_extract(self, url): - playlist_id = self._match_id(url) - - webpage = self._download_webpage(url, playlist_id) - - entries = [ - self.url_result( - self.BRIGHTCOVE_URL_TEMPLATE % video_id, 'BrightcoveNew', video_id) - for video_id in re.findall( - r'site\.brightcove\s*\([^,]+,\s*["\'](title_\d+)', webpage)] - - return self.playlist_result( - entries, playlist_id, - self._og_search_title(webpage, fatal=False), - self._og_search_description(webpage)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/extractors.py new/youtube-dl/youtube_dl/extractor/extractors.py --- old/youtube-dl/youtube_dl/extractor/extractors.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/extractors.py 2018-04-24 20:11:43.000000000 +0200 @@ -326,7 +326,6 @@ FiveThirtyEightIE, ) from .esri import EsriVideoIE -from .etonline import ETOnlineIE from .europa import EuropaIE from .everyonesmixtape import EveryonesMixtapeIE from .expotv import ExpoTVIE diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/extremetube.py new/youtube-dl/youtube_dl/extractor/extremetube.py --- old/youtube-dl/youtube_dl/extractor/extremetube.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/extremetube.py 2018-04-24 20:11:37.000000000 +0200 @@ -8,12 +8,12 @@ _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P<id>[^/#?&]+)' _TESTS = [{ 'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-c...', - 'md5': '1fb9228f5e3332ec8c057d6ac36f33e0', + 'md5': '92feaafa4b58e82f261e5419f39c60cb', 'info_dict': { 'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431', 'ext': 'mp4', 'title': 'Music Video 14 british euro brit european cumshots swallow', - 'uploader': 'unknown', + 'uploader': 'anonim', 'view_count': int, 'age_limit': 18, } @@ -36,10 +36,10 @@ r'<h1[^>]+title="([^"]+)"[^>]*>', webpage, 'title') uploader = self._html_search_regex( - r'Uploaded by:\s*</strong>\s*(.+?)\s*</div>', + r'Uploaded by:\s*</[^>]+>\s*<a[^>]+>(.+?)</a>', webpage, 'uploader', fatal=False) view_count = str_to_int(self._search_regex( - r'Views:\s*</strong>\s*<span>([\d,\.]+)</span>', + r'Views:\s*</[^>]+>\s*<[^>]+>([\d,\.]+)</', webpage, 'view count', fatal=False)) info.update({ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/generic.py new/youtube-dl/youtube_dl/extractor/generic.py --- old/youtube-dl/youtube_dl/extractor/generic.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/generic.py 2018-04-24 20:11:37.000000000 +0200 @@ -1220,7 +1220,7 @@ 'title': '35871', 'timestamp': 1355743100, 'upload_date': '20121217', - 'uploader_id': 'batchUser', + 'uploader_id': 'cplapp@learn360.com', }, 'add_ie': ['Kaltura'], }, @@ -1271,6 +1271,22 @@ }, 'add_ie': ['Kaltura'], }, + { + # meta twitter:player + 'url': 'http://thechive.com/2017/12/08/all-i-want-for-christmas-is-more-twerk/', + 'info_dict': { + 'id': '0_01b42zps', + 'ext': 'mp4', + 'title': 'Main Twerk (Video)', + 'upload_date': '20171208', + 'uploader_id': 'sebastian.salinas@thechive.com', + 'timestamp': 1512713057, + }, + 'params': { + 'skip_download': True, + }, + 'add_ie': ['Kaltura'], + }, # referrer protected EaglePlatform embed { 'url': 'https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/instagram.py new/youtube-dl/youtube_dl/extractor/instagram.py --- old/youtube-dl/youtube_dl/extractor/instagram.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/instagram.py 2018-04-24 20:11:37.000000000 +0200 @@ -6,11 +6,16 @@ import re from .common import InfoExtractor -from ..compat import compat_str +from ..compat import ( + compat_str, + compat_HTTPError, +) from ..utils import ( + ExtractorError, get_element_by_attribute, int_or_none, lowercase_escape, + std_headers, try_get, ) @@ -239,6 +244,8 @@ } } + _gis_tmpl = None + def _entries(self, data): def get_count(suffix): return int_or_none(try_get( @@ -254,19 +261,39 @@ for page_num in itertools.count(1): variables = json.dumps({ 'id': uploader_id, - 'first': 100, + 'first': 12, 'after': cursor, }) - s = '%s:%s:%s' % (rhx_gis, csrf_token, variables) - media = self._download_json( - 'https://www.instagram.com/graphql/query/', uploader_id, - 'Downloading JSON page %d' % page_num, headers={ - 'X-Requested-With': 'XMLHttpRequest', - 'X-Instagram-GIS': hashlib.md5(s.encode('utf-8')).hexdigest(), - }, query={ - 'query_hash': '472f257a40c653c64c666ce877d59d2b', - 'variables': variables, - })['data']['user']['edge_owner_to_timeline_media'] + + if self._gis_tmpl: + gis_tmpls = [self._gis_tmpl] + else: + gis_tmpls = [ + '%s' % rhx_gis, + '', + '%s:%s' % (rhx_gis, csrf_token), + '%s:%s:%s' % (rhx_gis, csrf_token, std_headers['User-Agent']), + ] + + for gis_tmpl in gis_tmpls: + try: + media = self._download_json( + 'https://www.instagram.com/graphql/query/', uploader_id, + 'Downloading JSON page %d' % page_num, headers={ + 'X-Requested-With': 'XMLHttpRequest', + 'X-Instagram-GIS': hashlib.md5( + ('%s:%s' % (gis_tmpl, variables)).encode('utf-8')).hexdigest(), + }, query={ + 'query_hash': '42323d64886122307be10013ad2dcc44', + 'variables': variables, + })['data']['user']['edge_owner_to_timeline_media'] + self._gis_tmpl = gis_tmpl + break + except ExtractorError as e: + if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403: + if gis_tmpl != gis_tmpls[-1]: + continue + raise edges = media.get('edges') if not edges or not isinstance(edges, list): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/kaltura.py new/youtube-dl/youtube_dl/extractor/kaltura.py --- old/youtube-dl/youtube_dl/extractor/kaltura.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/kaltura.py 2018-04-24 20:11:37.000000000 +0200 @@ -135,10 +135,10 @@ ''', webpage) or re.search( r'''(?xs) - <iframe[^>]+src=(?P<q1>["']) - (?:https?:)?//(?:www\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+) + <(?:iframe[^>]+src|meta[^>]+\bcontent)=(?P<q1>["']) + (?:https?:)?//(?:(?:www|cdnapi)\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+) (?:(?!(?P=q1)).)* - [?&]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+) + [?&;]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+) (?P=q1) ''', webpage) ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/keezmovies.py new/youtube-dl/youtube_dl/extractor/keezmovies.py --- old/youtube-dl/youtube_dl/extractor/keezmovies.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/keezmovies.py 2018-04-24 20:11:37.000000000 +0200 @@ -20,23 +20,23 @@ class KeezMoviesIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/(?:(?P<display_id>[^/]+)-)?(?P<id>\d+)' _TESTS = [{ - 'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-121...', - 'md5': '1c1e75d22ffa53320f45eeb07bc4cdc0', + 'url': 'https://www.keezmovies.com/video/arab-wife-want-it-so-bad-i-see-she-thirsty-...', + 'md5': '2ac69cdb882055f71d82db4311732a1a', 'info_dict': { - 'id': '1214711', - 'display_id': 'petite-asian-lady-mai-playing-in-bathtub', + 'id': '18070681', + 'display_id': 'arab-wife-want-it-so-bad-i-see-she-thirsty-and-has-tiny-money', 'ext': 'mp4', - 'title': 'Petite Asian Lady Mai Playing In Bathtub', - 'thumbnail': r're:^https?://.*\.jpg$', + 'title': 'Arab wife want it so bad I see she thirsty and has tiny money.', + 'thumbnail': None, 'view_count': int, 'age_limit': 18, } }, { - 'url': 'http://www.keezmovies.com/video/1214711', + 'url': 'http://www.keezmovies.com/video/18070681', 'only_matching': True, }] - def _extract_info(self, url): + def _extract_info(self, url, fatal=True): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') display_id = (mobj.group('display_id') @@ -55,7 +55,7 @@ encrypted = False def extract_format(format_url, height=None): - if not isinstance(format_url, compat_str) or not format_url.startswith('http'): + if not isinstance(format_url, compat_str) or not format_url.startswith(('http', '//')): return if format_url in format_urls: return @@ -105,7 +105,11 @@ raise ExtractorError( 'Video %s is no longer available' % video_id, expected=True) - self._sort_formats(formats) + try: + self._sort_formats(formats) + except ExtractorError: + if fatal: + raise if not title: title = self._html_search_regex( @@ -122,7 +126,9 @@ } def _real_extract(self, url): - webpage, info = self._extract_info(url) + webpage, info = self._extract_info(url, fatal=False) + if not info['formats']: + return self.url_result(url, 'Generic') info['view_count'] = str_to_int(self._search_regex( r'<b>([\d,.]+)</b> Views?', webpage, 'view count', fatal=False)) return info diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/mofosex.py new/youtube-dl/youtube_dl/extractor/mofosex.py --- old/youtube-dl/youtube_dl/extractor/mofosex.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/mofosex.py 2018-04-24 20:11:37.000000000 +0200 @@ -12,7 +12,7 @@ _VALID_URL = r'https?://(?:www\.)?mofosex\.com/videos/(?P<id>\d+)/(?P<display_id>[^/?#&.]+)\.html' _TESTS = [{ 'url': 'http://www.mofosex.com/videos/318131/amateur-teen-playing-and-masturbating-3...', - 'md5': '39a15853632b7b2e5679f92f69b78e91', + 'md5': '558fcdafbb63a87c019218d6e49daf8a', 'info_dict': { 'id': '318131', 'display_id': 'amateur-teen-playing-and-masturbating-318131', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/nexx.py new/youtube-dl/youtube_dl/extractor/nexx.py --- old/youtube-dl/youtube_dl/extractor/nexx.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/nexx.py 2018-04-24 20:11:37.000000000 +0200 @@ -230,15 +230,18 @@ azure_locator = stream_data['azureLocator'] - AZURE_URL = 'http://nx%s%02d.akamaized.net/' - - def get_cdn_shield_base(shield_type='', prefix='-p'): + def get_cdn_shield_base(shield_type='', static=False): for secure in ('', 's'): cdn_shield = stream_data.get('cdnShield%sHTTP%s' % (shield_type, secure.upper())) if cdn_shield: return 'http%s://%s' % (secure, cdn_shield) else: - return AZURE_URL % (prefix, int(stream_data['azureAccount'].replace('nexxplayplus', ''))) + if 'fb' in stream_data['azureAccount']: + prefix = 'df' if static else 'f' + else: + prefix = 'd' if static else 'p' + account = int(stream_data['azureAccount'].replace('nexxplayplus', '').replace('nexxplayfb', '')) + return 'http://nx-%s%02d.akamaized.net/' % (prefix, account) azure_stream_base = get_cdn_shield_base() is_ml = ',' in language @@ -260,7 +263,7 @@ formats.extend(self._extract_ism_formats( azure_manifest_url % '', video_id, ism_id='%s-mss' % cdn, fatal=False)) - azure_progressive_base = get_cdn_shield_base('Prog', '-d') + azure_progressive_base = get_cdn_shield_base('Prog', True) azure_file_distribution = stream_data.get('azureFileDistribution') if azure_file_distribution: fds = azure_file_distribution.split(',') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/nick.py new/youtube-dl/youtube_dl/extractor/nick.py --- old/youtube-dl/youtube_dl/extractor/nick.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/nick.py 2018-04-24 20:11:37.000000000 +0200 @@ -81,13 +81,23 @@ class NickBrIE(MTVServicesInfoExtractor): IE_NAME = 'nickelodeon:br' - _VALID_URL = r'https?://(?P<domain>(?:www\.)?nickjr|mundonick\.uol)\.com\.br/(?:programas/)?[^/]+/videos/(?:episodios/)?(?P<id>[^/?#.]+)' + _VALID_URL = r'''(?x) + https?:// + (?: + (?P<domain>(?:www\.)?nickjr|mundonick\.uol)\.com\.br| + (?:www\.)?nickjr\.nl + ) + /(?:programas/)?[^/]+/videos/(?:episodios/)?(?P<id>[^/?\#.]+) + ''' _TESTS = [{ 'url': 'http://www.nickjr.com.br/patrulha-canina/videos/210-labirinto-de-pipoca/', 'only_matching': True, }, { 'url': 'http://mundonick.uol.com.br/programas/the-loud-house/videos/muitas-irmas/7lj...', 'only_matching': True, + }, { + 'url': 'http://www.nickjr.nl/paw-patrol/videos/311-ge-wol-dig-om-terug-te-zijn/', + 'only_matching': True, }] def _real_extract(self, url): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/openload.py new/youtube-dl/youtube_dl/extractor/openload.py --- old/youtube-dl/youtube_dl/extractor/openload.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/openload.py 2018-04-24 20:11:43.000000000 +0200 @@ -340,7 +340,10 @@ get_element_by_id('streamurj', webpage) or self._search_regex( (r'>\s*([\w-]+~\d{10,}~\d+\.\d+\.0\.0~[\w-]+)\s*<', - r'>\s*([\w~-]+~\d+\.\d+\.\d+\.\d+~[\w~-]+)'), webpage, + r'>\s*([\w~-]+~\d+\.\d+\.\d+\.\d+~[\w~-]+)', + r'>\s*([\w-]+~\d{10,}~(?:[a-f\d]+:){2}:~[\w-]+)\s*<', + r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)\s*<', + r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)'), webpage, 'stream URL')) video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/pornflip.py new/youtube-dl/youtube_dl/extractor/pornflip.py --- old/youtube-dl/youtube_dl/extractor/pornflip.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/pornflip.py 2018-04-24 20:11:43.000000000 +0200 @@ -14,7 +14,7 @@ class PornFlipIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[0-9A-Za-z-]{11})' + _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[^/?#&]+)' _TESTS = [{ 'url': 'https://www.pornflip.com/v/wz7DfNhMmep', 'md5': '98c46639849145ae1fd77af532a9278c', @@ -40,6 +40,9 @@ }, { 'url': 'https://www.pornflip.com/embed/EkRD6-vS2-s', 'only_matching': True, + }, { + 'url': 'https://www.pornflip.com/v/NG9q6Pb_iK8', + 'only_matching': True, }] def _real_extract(self, url): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/rentv.py new/youtube-dl/youtube_dl/extractor/rentv.py --- old/youtube-dl/youtube_dl/extractor/rentv.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/rentv.py 2018-04-24 20:11:37.000000000 +0200 @@ -3,6 +3,10 @@ from .common import InfoExtractor from ..compat import compat_str +from ..utils import ( + determine_ext, + int_or_none, +) class RENTVIE(InfoExtractor): @@ -13,7 +17,9 @@ 'info_dict': { 'id': '118577', 'ext': 'mp4', - 'title': 'Документальный спецпроект: "Промывка мозгов. Технологии XXI века"' + 'title': 'Документальный спецпроект: "Промывка мозгов. Технологии XXI века"', + 'timestamp': 1472230800, + 'upload_date': '20160826', } }, { 'url': 'http://ren.tv/player/118577', @@ -26,9 +32,33 @@ def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage('http://ren.tv/player/' + video_id, video_id) - jw_config = self._parse_json(self._search_regex( - r'config\s*=\s*({.+});', webpage, 'jw config'), video_id) - return self._parse_jwplayer_data(jw_config, video_id, m3u8_id='hls') + config = self._parse_json(self._search_regex( + r'config\s*=\s*({.+})\s*;', webpage, 'config'), video_id) + title = config['title'] + formats = [] + for video in config['src']: + src = video.get('src') + if not src or not isinstance(src, compat_str): + continue + ext = determine_ext(src) + if ext == 'm3u8': + formats.extend(self._extract_m3u8_formats( + src, video_id, 'mp4', entry_protocol='m3u8_native', + m3u8_id='hls', fatal=False)) + else: + formats.append({ + 'url': src, + }) + self._sort_formats(formats) + return { + 'id': video_id, + 'title': title, + 'description': config.get('description'), + 'thumbnail': config.get('image'), + 'duration': int_or_none(config.get('duration')), + 'timestamp': int_or_none(config.get('date')), + 'formats': formats, + } class RENTVArticleIE(InfoExtractor): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/svt.py new/youtube-dl/youtube_dl/extractor/svt.py --- old/youtube-dl/youtube_dl/extractor/svt.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/svt.py 2018-04-24 20:11:37.000000000 +0200 @@ -22,6 +22,8 @@ _GEO_COUNTRIES = ['SE'] def _extract_video(self, video_info, video_id): + is_live = dict_get(video_info, ('live', 'simulcast'), default=False) + m3u8_protocol = 'm3u8' if is_live else 'm3u8_native' formats = [] for vr in video_info['videoReferences']: player_type = vr.get('playerType') or vr.get('format') @@ -30,7 +32,7 @@ if ext == 'm3u8': formats.extend(self._extract_m3u8_formats( vurl, video_id, - ext='mp4', entry_protocol='m3u8_native', + ext='mp4', entry_protocol=m3u8_protocol, m3u8_id=player_type, fatal=False)) elif ext == 'f4m': formats.extend(self._extract_f4m_formats( @@ -90,6 +92,7 @@ 'season_number': season_number, 'episode': episode, 'episode_number': episode_number, + 'is_live': is_live, } @@ -134,7 +137,7 @@ class SVTPlayIE(SVTPlayBaseIE): IE_DESC = 'SVT Play and Öppet arkiv' - _VALID_URL = r'https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp)/(?P<id>[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp|kanaler)/(?P<id>[^/?#&]+)' _TESTS = [{ 'url': 'http://www.svtplay.se/video/5996901/flygplan-till-haile-selassie/flygplan-ti...', 'md5': '2b6704fe4a28801e1a098bbf3c5ac611', @@ -158,6 +161,9 @@ }, { 'url': 'http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg', 'only_matching': True, + }, { + 'url': 'https://www.svtplay.se/kanaler/svt1', + 'only_matching': True, }] def _real_extract(self, url): @@ -173,6 +179,10 @@ thumbnail = self._og_search_thumbnail(webpage) + def adjust_title(info): + if info['is_live']: + info['title'] = self._live_title(info['title']) + if data: video_info = try_get( data, lambda x: x['context']['dispatcher']['stores']['VideoTitlePageStore']['data']['video'], @@ -183,6 +193,7 @@ 'title': data['context']['dispatcher']['stores']['MetaStore']['title'], 'thumbnail': thumbnail, }) + adjust_title(info_dict) return info_dict video_id = self._search_regex( @@ -198,6 +209,7 @@ info_dict['title'] = re.sub( r'\s*\|\s*.+?$', '', info_dict.get('episode') or self._og_search_title(webpage)) + adjust_title(info_dict) return info_dict diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/twitch.py new/youtube-dl/youtube_dl/extractor/twitch.py --- old/youtube-dl/youtube_dl/extractor/twitch.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/twitch.py 2018-04-24 20:11:43.000000000 +0200 @@ -168,6 +168,13 @@ return self.playlist_result(entries, info['id'], info['title']) def _extract_info(self, info): + status = info.get('status') + if status == 'recording': + is_live = True + elif status == 'recorded': + is_live = False + else: + is_live = None return { 'id': info['_id'], 'title': info.get('title') or 'Untitled Broadcast', @@ -178,6 +185,7 @@ 'uploader_id': info.get('channel', {}).get('name'), 'timestamp': parse_iso8601(info.get('recorded_at')), 'view_count': int_or_none(info.get('views')), + 'is_live': is_live, } def _real_extract(self, url): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/youtube.py new/youtube-dl/youtube_dl/extractor/youtube.py --- old/youtube-dl/youtube_dl/extractor/youtube.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/youtube.py 2018-04-24 20:11:37.000000000 +0200 @@ -87,7 +87,7 @@ (username, password) = self._get_login_info() # No authentication to be performed if username is None: - if self._LOGIN_REQUIRED: + if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None: raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True) return True @@ -2699,10 +2699,7 @@ def _real_initialize(self): self._login() - def _real_extract(self, url): - page = self._download_webpage( - 'https://www.youtube.com/feed/%s' % self._FEED_NAME, self._PLAYLIST_TITLE) - + def _entries(self, page): # The extraction process is the same as for playlists, but the regex # for the video ids doesn't contain an index ids = [] @@ -2713,12 +2710,15 @@ # 'recommended' feed has infinite 'load more' and each new portion spins # the same videos in (sometimes) slightly different order, so we'll check # for unicity and break when portion has no new videos - new_ids = filter(lambda video_id: video_id not in ids, orderedSet(matches)) + new_ids = list(filter(lambda video_id: video_id not in ids, orderedSet(matches))) if not new_ids: break ids.extend(new_ids) + for entry in self._ids_to_results(new_ids): + yield entry + mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html) if not mobj: break @@ -2730,8 +2730,12 @@ content_html = more['content_html'] more_widget_html = more['load_more_widget_html'] + def _real_extract(self, url): + page = self._download_webpage( + 'https://www.youtube.com/feed/%s' % self._FEED_NAME, + self._PLAYLIST_TITLE) return self.playlist_result( - self._ids_to_results(ids), playlist_title=self._PLAYLIST_TITLE) + self._entries(page), playlist_title=self._PLAYLIST_TITLE) class YoutubeWatchLaterIE(YoutubePlaylistIE): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/utils.py new/youtube-dl/youtube_dl/utils.py --- old/youtube-dl/youtube_dl/utils.py 2018-04-09 00:29:45.000000000 +0200 +++ new/youtube-dl/youtube_dl/utils.py 2018-04-24 20:11:43.000000000 +0200 @@ -2574,8 +2574,8 @@ return op(actual_value, comparison_value) UNARY_OPERATORS = { - '': lambda v: v is not None, - '!': lambda v: v is None, + '': lambda v: (v is True) if isinstance(v, bool) else (v is not None), + '!': lambda v: (v is False) if isinstance(v, bool) else (v is None), } operator_rex = re.compile(r'''(?x)\s* (?P<op>%s)\s*(?P<key>[a-z_]+) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/version.py new/youtube-dl/youtube_dl/version.py --- old/youtube-dl/youtube_dl/version.py 2018-04-15 20:09:12.000000000 +0200 +++ new/youtube-dl/youtube_dl/version.py 2018-04-24 20:12:36.000000000 +0200 @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2018.04.16' +__version__ = '2018.04.25'
participants (1)
-
root＠hilbert.suse.de