Mailinglist Archive: opensuse-commit (1903 mails)

< Previous Next >
commit you-get for openSUSE:Factory
Hello community,

here is the log from the commit of package you-get for openSUSE:Factory checked
in at 2019-04-26 22:54:32
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/you-get (Old)
and /work/SRC/openSUSE:Factory/.you-get.new.5536 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Package is "you-get"

Fri Apr 26 22:54:32 2019 rev:21 rq:697867 version:0.4.1295

Changes:
--------
--- /work/SRC/openSUSE:Factory/you-get/you-get.changes 2019-03-26
15:41:38.612270662 +0100
+++ /work/SRC/openSUSE:Factory/.you-get.new.5536/you-get.changes
2019-04-26 22:54:32.737311443 +0200
@@ -1,0 +2,5 @@
+Thu Apr 25 14:37:38 UTC 2019 - Luigi Baldoni <aloisio@xxxxxxx>
+
+- Update to version 0.4.1295 (no changelog supplied)
+
+-------------------------------------------------------------------

Old:
----
you-get-0.4.1270.tar.gz

New:
----
you-get-0.4.1295.tar.gz

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Other differences:
------------------
++++++ you-get.spec ++++++
--- /var/tmp/diff_new_pack.OANZw9/_old 2019-04-26 22:54:33.245311116 +0200
+++ /var/tmp/diff_new_pack.OANZw9/_new 2019-04-26 22:54:33.245311116 +0200
@@ -17,7 +17,7 @@


Name: you-get
-Version: 0.4.1270
+Version: 0.4.1295
Release: 0
Summary: Dumb downloader that scrapes the web
License: MIT

++++++ you-get-0.4.1270.tar.gz -> you-get-0.4.1295.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/.travis.yml
new/you-get-0.4.1295/.travis.yml
--- old/you-get-0.4.1270/.travis.yml 2019-03-08 23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/.travis.yml 2019-04-25 16:23:50.000000000 +0200
@@ -6,15 +6,20 @@
- "3.4"
- "3.5"
- "3.6"
- - "3.7-dev"
- - "nightly"
- "pypy3"
+matrix:
+ include:
+ - python: "3.7"
+ dist: xenial
+ - python: "3.8-dev"
+ dist: xenial
+ - python: "nightly"
+ dist: xenial
before_install:
- if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'*
]]; then pip install flake8; fi
before_script:
- - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'*
]]; then flake8 . --count --select=E901,E999,F821,F822,F823 --show-source
--statistics; fi
+ - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'*
]]; then flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics; fi
script: make test
-sudo: false
notifications:
webhooks:
urls:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/common.py
new/you-get-0.4.1295/src/you_get/common.py
--- old/you-get-0.4.1270/src/you_get/common.py 2019-03-08 23:20:06.000000000
+0100
+++ new/you-get-0.4.1295/src/you_get/common.py 2019-04-25 16:23:50.000000000
+0200
@@ -10,6 +10,7 @@
import locale
import logging
import argparse
+import ssl
from http import cookiejar
from importlib import import_module
from urllib import request, parse, error
@@ -135,6 +136,7 @@
cookies = None
output_filename = None
auto_rename = False
+insecure = False

fake_headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', # noqa
@@ -391,7 +393,14 @@
retry_time = 3
for i in range(retry_time):
try:
- return request.urlopen(*args, **kwargs)
+ if insecure:
+ # ignore ssl errors
+ ctx = ssl.create_default_context()
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+ return request.urlopen(*args, context=ctx, **kwargs)
+ else:
+ return request.urlopen(*args, **kwargs)
except socket.timeout as e:
logging.debug('request attempt %s timeout' % str(i + 1))
if i + 1 == retry_time:
@@ -1489,6 +1498,11 @@
help='Auto rename same name different files'
)

+ download_grp.add_argument(
+ '-k', '--insecure', action='store_true', default=False,
+ help='ignore ssl errors'
+ )
+
proxy_grp = parser.add_argument_group('Proxy options')
proxy_grp = proxy_grp.add_mutually_exclusive_group()
proxy_grp.add_argument(
@@ -1533,7 +1547,7 @@
global extractor_proxy
global output_filename
global auto_rename
-
+ global insecure
output_filename = args.output_filename
extractor_proxy = args.extractor_proxy

@@ -1561,6 +1575,11 @@
player = args.player
caption = False

+ if args.insecure:
+ # ignore ssl
+ insecure = True
+
+
if args.no_proxy:
set_http_proxy('')
else:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/acfun.py
new/you-get-0.4.1295/src/you_get/extractors/acfun.py
--- old/you-get-0.4.1270/src/you_get/extractors/acfun.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/acfun.py 2019-04-25
16:23:50.000000000 +0200
@@ -121,7 +121,7 @@
# bangumi
elif re.match("http://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
html = get_content(url)
- title = match1(html, r'"newTitle"\s*:\s*"([^"]+)"')
+ title = match1(html, r'"title"\s*:\s*"([^"]+)"')
if match1(url, r'_(\d+)$'): # current P
title = title + " " + r1(r'active">([^<]*)', html)
vid = match1(html, r'videoId="(\d+)"')
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/baidu.py
new/you-get-0.4.1295/src/you_get/extractors/baidu.py
--- old/you-get-0.4.1270/src/you_get/extractors/baidu.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/baidu.py 2019-04-25
16:23:50.000000000 +0200
@@ -38,7 +38,7 @@

def baidu_get_song_lyric(data):
lrc = data['lrcLink']
- return None if lrc is '' else "http://music.baidu.com%s"; % lrc
+ return "http://music.baidu.com%s"; % lrc if lrc else None


def baidu_download_song(sid, output_dir='.', merge=True, info_only=False):
@@ -123,7 +123,7 @@
elif re.match('http://tieba.baidu.com/', url):
try:
# embedded videos
- embed_download(url, output_dir, merge=merge, info_only=info_only)
+ embed_download(url, output_dir, merge=merge, info_only=info_only,
**kwargs)
except:
# images
html = get_html(url)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/bilibili.py
new/you-get-0.4.1295/src/you_get/extractors/bilibili.py
--- old/you-get-0.4.1270/src/you_get/extractors/bilibili.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/bilibili.py 2019-04-25
16:23:50.000000000 +0200
@@ -12,7 +12,8 @@
stream_types = [
{'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,
'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清
1080P60'},
- # 'id': 'hdflv2', 'quality': 112?
+ {'id': 'hdflv2', 'quality': 112, 'audio_quality': 30280,
+ 'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P+'},
{'id': 'flv', 'quality': 80, 'audio_quality': 30280,
'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P'},
{'id': 'flv720_p60', 'quality': 74, 'audio_quality': 30280,
@@ -111,6 +112,13 @@
def bilibili_vc_api(video_id):
return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s'
% video_id

+ @staticmethod
+ def url_size(url, faker=False, headers={},err_value=0):
+ try:
+ return url_size(url,faker,headers)
+ except:
+ return err_value
+
def prepare(self, **kwargs):
self.stream_qualities = {s['quality']: s for s in self.stream_types}

@@ -244,7 +252,7 @@
desc = s['desc']
audio_quality = s['audio_quality']
baseurl = video['baseUrl']
- size = url_size(baseurl,
headers=self.bilibili_headers(referer=self.url))
+ size = self.url_size(baseurl,
headers=self.bilibili_headers(referer=self.url))

# find matching audio track
audio_baseurl =
playinfo['data']['dash']['audio'][0]['baseUrl']
@@ -253,7 +261,7 @@
audio_baseurl = audio['baseUrl']
break
if not audio_size_cache.get(audio_quality, False):
- audio_size_cache[audio_quality] =
url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+ audio_size_cache[audio_quality] =
self.url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
size += audio_size_cache[audio_quality]

self.dash_streams[format_id] = {'container':
container, 'quality': desc,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/douyutv.py
new/you-get-0.4.1295/src/you_get/extractors/douyutv.py
--- old/you-get-0.4.1270/src/you_get/extractors/douyutv.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/douyutv.py 2019-04-25
16:23:50.000000000 +0200
@@ -62,12 +62,12 @@
json_content = json.loads(content)
data = json_content['data']
server_status = json_content.get('error', 0)
- if server_status is not 0:
+ if server_status != 0:
raise ValueError("Server returned error:%s" % server_status)

title = data.get('room_name')
show_status = data.get('show_status')
- if show_status is not "1":
+ if show_status != "1":
raise ValueError("The live stream is not online! (Errno:%s)" %
server_status)

real_url = data.get('rtmp_url') + '/' + data.get('rtmp_live')
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/embed.py
new/you-get-0.4.1295/src/you_get/extractors/embed.py
--- old/you-get-0.4.1270/src/you_get/extractors/embed.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/embed.py 2019-04-25
16:23:50.000000000 +0200
@@ -67,7 +67,7 @@
recur_limit = 3


-def embed_download(url, output_dir = '.', merge = True, info_only = False
,**kwargs):
+def embed_download(url, output_dir = '.', merge = True, info_only = False,
**kwargs):
content = get_content(url, headers=fake_headers)
found = False
title = match1(content, '<title>([^<>]+)</title>')
@@ -75,43 +75,43 @@
vids = matchall(content, youku_embed_patterns)
for vid in set(vids):
found = True
- youku_download_by_vid(vid, title=title, output_dir=output_dir,
merge=merge, info_only=info_only)
+ youku_download_by_vid(vid, title=title, output_dir=output_dir,
merge=merge, info_only=info_only, **kwargs)

vids = matchall(content, tudou_embed_patterns)
for vid in set(vids):
found = True
- tudou_download_by_id(vid, title=title, output_dir=output_dir,
merge=merge, info_only=info_only)
+ tudou_download_by_id(vid, title=title, output_dir=output_dir,
merge=merge, info_only=info_only, **kwargs)

vids = matchall(content, yinyuetai_embed_patterns)
for vid in vids:
found = True
- yinyuetai_download_by_id(vid, title=title, output_dir=output_dir,
merge=merge, info_only=info_only)
+ yinyuetai_download_by_id(vid, title=title, output_dir=output_dir,
merge=merge, info_only=info_only, **kwargs)

vids = matchall(content, iqiyi_embed_patterns)
for vid in vids:
found = True
- iqiyi_download_by_vid((vid[1], vid[0]), title=title,
output_dir=output_dir, merge=merge, info_only=info_only)
+ iqiyi_download_by_vid((vid[1], vid[0]), title=title,
output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)

urls = matchall(content, netease_embed_patterns)
for url in urls:
found = True
- netease_download(url, output_dir=output_dir, merge=merge,
info_only=info_only)
+ netease_download(url, output_dir=output_dir, merge=merge,
info_only=info_only, **kwargs)

urls = matchall(content, vimeo_embed_patters)
for url in urls:
found = True
- vimeo_download_by_id(url, title=title, output_dir=output_dir,
merge=merge, info_only=info_only, referer=url)
+ vimeo_download_by_id(url, title=title, output_dir=output_dir,
merge=merge, info_only=info_only, referer=url, **kwargs)

urls = matchall(content, dailymotion_embed_patterns)
for url in urls:
found = True
- dailymotion_download(url, output_dir=output_dir, merge=merge,
info_only=info_only)
+ dailymotion_download(url, output_dir=output_dir, merge=merge,
info_only=info_only, **kwargs)

aids = matchall(content, bilibili_embed_patterns)
for aid in aids:
found = True
url = 'http://www.bilibili.com/video/av%s/' % aid
- bilibili_download(url, output_dir=output_dir, merge=merge,
info_only=info_only)
+ bilibili_download(url, output_dir=output_dir, merge=merge,
info_only=info_only, **kwargs)

iqiyi_urls = matchall(content, iqiyi_patterns)
for url in iqiyi_urls:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/facebook.py
new/you-get-0.4.1295/src/you_get/extractors/facebook.py
--- old/you-get-0.4.1270/src/you_get/extractors/facebook.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/facebook.py 2019-04-25
16:23:50.000000000 +0200
@@ -6,6 +6,7 @@
import json

def facebook_download(url, output_dir='.', merge=True, info_only=False,
**kwargs):
+ url = re.sub(r'//.*?facebook.com','//facebook.com',url)
html = get_html(url)

title = r1(r'<title id="pageTitle">(.+)</title>', html)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/iqiyi.py
new/you-get-0.4.1295/src/you_get/extractors/iqiyi.py
--- old/you-get-0.4.1270/src/you_get/extractors/iqiyi.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/iqiyi.py 2019-04-25
16:23:50.000000000 +0200
@@ -136,12 +136,9 @@
r1(r'vid=([^&]+)', self.url) or \
r1(r'data-player-videoid="([^"]+)"', html) or
r1(r'vid=(.+?)\&', html) or r1(r'param\[\'vid\'\]\s*=\s*"(.+?)"', html)
self.vid = (tvid, videoid)
- info_u = 'http://mixer.video.iqiyi.com/jp/mixin/videos/' + tvid
- mixin = get_content(info_u)
- mixin_json = json.loads(mixin[len('var tvInfoJs='):])
- real_u = mixin_json['url']
- real_html = get_content(real_u)
- self.title = match1(real_html, '<title>([^<]+)').split('-')[0]
+ info_u =
'http://pcw-api.iqiyi.com/video/video/playervideoinfo?tvid=' + tvid
+ json_res = get_content(info_u)
+ self.title = json.loads(json_res)['data']['vn']
tvid, videoid = self.vid
info = getVMS(tvid, videoid)
assert info['code'] == 'A00000', "can't play this video"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/panda.py
new/you-get-0.4.1295/src/you_get/extractors/panda.py
--- old/you-get-0.4.1270/src/you_get/extractors/panda.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/panda.py 2019-04-25
16:23:50.000000000 +0200
@@ -15,7 +15,7 @@
json_request_url
="http://www.panda.tv/api_room_v2?roomid={}&__plat=pc_web&_={}".format(roomid,
int(time.time()))
content = get_html(json_request_url)
api_json = json.loads(content)
-
+
errno = api_json["errno"]
errmsg = api_json["errmsg"]
if errno:
@@ -25,7 +25,7 @@
room_key = data["videoinfo"]["room_key"]
plflag = data["videoinfo"]["plflag"].split("_")
status = data["videoinfo"]["status"]
- if status is not "2":
+ if status != "2":
raise ValueError("The live stream is not online! (status:%s)" % status)

data2 = json.loads(data["videoinfo"]["plflag_list"])
@@ -33,7 +33,7 @@
sign = data2["auth"]["sign"]
ts = data2["auth"]["time"]
real_url =
"http://pl{}.live.panda.tv/live_panda/{}.flv?sign={}&ts={}&rid={}".format(plflag[1],
room_key, sign, ts, rid)
-
+
print_info(site_info, title, 'flv', float('inf'))
if not info_only:
download_urls([real_url], title, 'flv', None, output_dir, merge =
merge)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/tiktok.py
new/you-get-0.4.1295/src/you_get/extractors/tiktok.py
--- old/you-get-0.4.1270/src/you_get/extractors/tiktok.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/tiktok.py 2019-04-25
16:23:50.000000000 +0200
@@ -6,12 +6,10 @@

def tiktok_download(url, output_dir='.', merge=True, info_only=False,
**kwargs):
html = get_html(url, faker=True)
- title = r1(r'<title>(.*?)</title>', html)
+ title = r1(r'<title.*?>(.*?)</title>', html)
video_id = r1(r'/video/(\d+)', url) or r1(r'musical\?id=(\d+)', html)
title = '%s [%s]' % (title, video_id)
- dataText = r1(r'var data = \[(.*)\] ', html) or r1(r'var data = (\{.*\})',
html)
- data = json.loads(dataText)
- source = 'http:' + data['video']['play_addr']['url_list'][0]
+ source = r1(r'<video .*?src="([^"]+)"', html)
mime, ext, size = url_info(source)

print_info(site_info, title, mime, size)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/toutiao.py
new/you-get-0.4.1295/src/you_get/extractors/toutiao.py
--- old/you-get-0.4.1270/src/you_get/extractors/toutiao.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/toutiao.py 2019-04-25
16:23:50.000000000 +0200
@@ -1,27 +1,36 @@
#!/usr/bin/env python
-import base64
-
import binascii
-
-from ..common import *
import random
from json import loads
+from urllib.parse import urlparse
+
+from ..common import *
+
+try:
+ from base64 import decodebytes
+except ImportError:
+ from base64 import decodestring
+
+ decodebytes = decodestring

__all__ = ['toutiao_download', ]


-def sign_video_url(vid):
- # some code from http://codecloud.net/110854.html
- r = str(random.random())[2:]
+def random_with_n_digits(n):
+ return random.randint(10 ** (n - 1), (10 ** n) - 1)

- def right_shift(val, n):
- return val >> n if val >= 0 else (val + 0x100000000) >> n

- url = 'http://i.snssdk.com/video/urls/v/1/toutiao/mp4/%s' % vid
- n = url.replace("http://i.snssdk.com";, "")+ '?r=' + r
- c = binascii.crc32(n.encode("ascii"))
- s = right_shift(c, 0)
- return url + '?r=%s&s=%s' % (r, s)
+def sign_video_url(vid):
+ r = str(random_with_n_digits(16))
+
+ url =
'https://ib.365yg.com/video/urls/v/1/toutiao/mp4/{vid}'.format(vid=vid)
+ n = urlparse(url).path + '?r=' + r
+ b_n = bytes(n, encoding="utf-8")
+ s = binascii.crc32(b_n)
+ aid = 1364
+ ts = int(time.time() * 1000)
+ return url +
'?r={r}&s={s}&aid={aid}&vfrom=xgplayer&callback=axiosJsonpCallback1&_={ts}'.format(r=r,
s=s, aid=aid,
+
ts=ts)


class ToutiaoVideoInfo(object):
@@ -43,12 +52,12 @@
vRet = []
url = sign_video_url(video_id)
ret = get_content(url)
- ret = loads(ret)
+ ret = loads(ret[20:-1])
vlist = ret.get('data').get('video_list')
if len(vlist) > 0:
vInfo = vlist.get(sorted(vlist.keys(), reverse=True)[0])
vUrl = vInfo.get('main_url')
- vUrl = base64.decodestring(vUrl.encode('ascii')).decode('ascii')
+ vUrl = decodebytes(vUrl.encode('ascii')).decode('ascii')
videoInfo = ToutiaoVideoInfo()
videoInfo.bitrate = vInfo.get('bitrate')
videoInfo.definition = vInfo.get('definition')
@@ -63,8 +72,8 @@

def toutiao_download(url, output_dir='.', merge=True, info_only=False,
**kwargs):
html = get_html(url, faker=True)
- video_id = match1(html, r"videoid\s*:\s*'([^']+)',\n")
- title = match1(html, r"title: '([^']+)'.replace")
+ video_id = match1(html, r".*?videoId: '(?P<vid>.*)'")
+ title = match1(html, '.*?<title>(?P<title>.*?)</title>')
video_file_list = get_file_by_vid(video_id) # 调api获取视频源文件
type, ext, size = url_info(video_file_list[0].url, faker=True)
print_info(site_info=site_info, title=title, type=type, size=size)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/tumblr.py
new/you-get-0.4.1295/src/you_get/extractors/tumblr.py
--- old/you-get-0.4.1270/src/you_get/extractors/tumblr.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/tumblr.py 2019-04-25
16:23:50.000000000 +0200
@@ -49,9 +49,14 @@

tuggles = {}
for url in urls:
- hd_url = r1(r'(.+)_\d+\.jpg$', url) # FIXME: .png and .gif
- if hd_url is None: continue
- hd_url = hd_url + '_1280.jpg' # FIXME: decide actual quality
+ if url.endswith('.gif'):
+ hd_url = url
+ elif url.endswith('.jpg'):
+ hd_url = r1(r'(.+)_\d+\.jpg$', url) + '_1280.jpg' # FIXME:
decide actual quality
+ elif url.endswith('.png'):
+ hd_url = r1(r'(.+)_\d+\.png$', url) + '_1280.png' # FIXME:
decide actual quality
+ else:
+ continue
filename = parse.unquote(hd_url.split('/')[-1])
title = '.'.join(filename.split('.')[:-1])
tumblr_id = r1(r'^tumblr_(.+)_\d+$', title)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/twitter.py
new/you-get-0.4.1295/src/you_get/extractors/twitter.py
--- old/you-get-0.4.1270/src/you_get/extractors/twitter.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/twitter.py 2019-04-25
16:23:50.000000000 +0200
@@ -3,6 +3,7 @@
__all__ = ['twitter_download']

from ..common import *
+from .universal import *
from .vine import vine_download

def extract_m3u(source):
@@ -15,6 +16,10 @@
return ['https://video.twimg.com%s' % i for i in s2]

def twitter_download(url, output_dir='.', merge=True, info_only=False,
**kwargs):
+ if re.match(r'https?://pbs\.twimg\.com', url):
+ universal_download(url, output_dir, merge=merge, info_only=info_only,
**kwargs)
+ return
+
if re.match(r'https?://mobile', url): # normalize mobile URL
url = 'https://' + match1(url, r'//mobile\.(.+)')

diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/youtube.py
new/you-get-0.4.1295/src/you_get/extractors/youtube.py
--- old/you-get-0.4.1270/src/you_get/extractors/youtube.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/youtube.py 2019-04-25
16:23:50.000000000 +0200
@@ -195,7 +195,11 @@
self.download_playlist_by_url(self.url, **kwargs)
exit(0)

- video_info =
parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}'.format(self.vid)))
+ # Get video info
+ # 'eurl' is a magic parameter that can bypass age restriction
+ # full form:
'eurl=https%3A%2F%2Fyoutube.googleapis.com%2Fv%2F{VIDEO_ID}'
+ video_info =
parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}&eurl=https%3A%2F%2Fy'.format(self.vid)))
+ logging.debug('STATUS: %s' % video_info['status'][0])

ytplayer_config = None
if 'status' not in video_info:
@@ -214,7 +218,10 @@
stream_list =
ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
except:
stream_list =
video_info['url_encoded_fmt_stream_map'][0].split(',')
- self.html5player = None
+ if re.search('([^"]*/base\.js)"', video_page):
+ self.html5player = 'https://www.youtube.com' +
re.search('([^"]*/base\.js)"', video_page).group(1)
+ else:
+ self.html5player = None

else:
# Parse video page instead
@@ -226,7 +233,9 @@
stream_list =
ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')

elif video_info['status'] == ['fail']:
+ logging.debug('ERRORCODE: %s' % video_info['errorcode'][0])
if video_info['errorcode'] == ['150']:
+ # FIXME: still relevant?
if cookies:
# Load necessary cookies into headers (for age-restricted
videos)
consent, ssid, hsid, sid = 'YES', '', '', ''
@@ -404,74 +413,82 @@
# VEVO
if not self.html5player: return
self.js = get_content(self.html5player)
- if 'adaptive_fmts' in ytplayer_config['args']:
+
+ try:
+ # Video info from video page (not always available)
streams = [dict([(i.split('=')[0],
parse.unquote(i.split('=')[1]))
for i in afmt.split('&')])
for afmt in
ytplayer_config['args']['adaptive_fmts'].split(',')]
- for stream in streams: # get over speed limiting
- stream['url'] += '&ratebypass=yes'
- for stream in streams: # audio
- if stream['type'].startswith('audio/mp4'):
- dash_mp4_a_url = stream['url']
+ except:
+ streams = [dict([(i.split('=')[0],
+ parse.unquote(i.split('=')[1]))
+ for i in afmt.split('&')])
+ for afmt in
video_info['adaptive_fmts'][0].split(',')]
+
+ for stream in streams: # get over speed limiting
+ stream['url'] += '&ratebypass=yes'
+ for stream in streams: # audio
+ if stream['type'].startswith('audio/mp4'):
+ dash_mp4_a_url = stream['url']
+ if 's' in stream:
+ sig = self.__class__.decipher(self.js, stream['s'])
+ dash_mp4_a_url += '&signature={}'.format(sig)
+ dash_mp4_a_size = stream['clen']
+ elif stream['type'].startswith('audio/webm'):
+ dash_webm_a_url = stream['url']
+ if 's' in stream:
+ sig = self.__class__.decipher(self.js, stream['s'])
+ dash_webm_a_url += '&signature={}'.format(sig)
+ dash_webm_a_size = stream['clen']
+ for stream in streams: # video
+ if 'size' in stream:
+ if stream['type'].startswith('video/mp4'):
+ mimeType = 'video/mp4'
+ dash_url = stream['url']
if 's' in stream:
sig = self.__class__.decipher(self.js, stream['s'])
- dash_mp4_a_url += '&signature={}'.format(sig)
- dash_mp4_a_size = stream['clen']
- elif stream['type'].startswith('audio/webm'):
- dash_webm_a_url = stream['url']
+ dash_url += '&signature={}'.format(sig)
+ dash_size = stream['clen']
+ itag = stream['itag']
+ dash_urls = self.__class__.chunk_by_range(dash_url,
int(dash_size))
+ dash_mp4_a_urls =
self.__class__.chunk_by_range(dash_mp4_a_url, int(dash_mp4_a_size))
+ self.dash_streams[itag] = {
+ 'quality': '%s (%s)' % (stream['size'],
stream['quality_label']),
+ 'itag': itag,
+ 'type': mimeType,
+ 'mime': mimeType,
+ 'container': 'mp4',
+ 'src': [dash_urls, dash_mp4_a_urls],
+ 'size': int(dash_size) + int(dash_mp4_a_size)
+ }
+ elif stream['type'].startswith('video/webm'):
+ mimeType = 'video/webm'
+ dash_url = stream['url']
if 's' in stream:
sig = self.__class__.decipher(self.js, stream['s'])
- dash_webm_a_url += '&signature={}'.format(sig)
- dash_webm_a_size = stream['clen']
- for stream in streams: # video
- if 'size' in stream:
- if stream['type'].startswith('video/mp4'):
- mimeType = 'video/mp4'
- dash_url = stream['url']
- if 's' in stream:
- sig = self.__class__.decipher(self.js,
stream['s'])
- dash_url += '&signature={}'.format(sig)
- dash_size = stream['clen']
- itag = stream['itag']
- dash_urls =
self.__class__.chunk_by_range(dash_url, int(dash_size))
- dash_mp4_a_urls =
self.__class__.chunk_by_range(dash_mp4_a_url, int(dash_mp4_a_size))
- self.dash_streams[itag] = {
- 'quality': '%s (%s)' % (stream['size'],
stream['quality_label']),
- 'itag': itag,
- 'type': mimeType,
- 'mime': mimeType,
- 'container': 'mp4',
- 'src': [dash_urls, dash_mp4_a_urls],
- 'size': int(dash_size) + int(dash_mp4_a_size)
- }
- elif stream['type'].startswith('video/webm'):
- mimeType = 'video/webm'
- dash_url = stream['url']
- if 's' in stream:
- sig = self.__class__.decipher(self.js,
stream['s'])
- dash_url += '&signature={}'.format(sig)
- dash_size = stream['clen']
- itag = stream['itag']
- audio_url = None
- audio_size = None
- try:
- audio_url = dash_webm_a_url
- audio_size = int(dash_webm_a_size)
- except UnboundLocalError as e:
- audio_url = dash_mp4_a_url
- audio_size = int(dash_mp4_a_size)
- dash_urls =
self.__class__.chunk_by_range(dash_url, int(dash_size))
- audio_urls =
self.__class__.chunk_by_range(audio_url, int(audio_size))
- self.dash_streams[itag] = {
- 'quality': '%s (%s)' % (stream['size'],
stream['quality_label']),
- 'itag': itag,
- 'type': mimeType,
- 'mime': mimeType,
- 'container': 'webm',
- 'src': [dash_urls, audio_urls],
- 'size': int(dash_size) + int(audio_size)
- }
+ dash_url += '&signature={}'.format(sig)
+ dash_size = stream['clen']
+ itag = stream['itag']
+ audio_url = None
+ audio_size = None
+ try:
+ audio_url = dash_webm_a_url
+ audio_size = int(dash_webm_a_size)
+ except UnboundLocalError as e:
+ audio_url = dash_mp4_a_url
+ audio_size = int(dash_mp4_a_size)
+ dash_urls = self.__class__.chunk_by_range(dash_url,
int(dash_size))
+ audio_urls = self.__class__.chunk_by_range(audio_url,
int(audio_size))
+ self.dash_streams[itag] = {
+ 'quality': '%s (%s)' % (stream['size'],
stream['quality_label']),
+ 'itag': itag,
+ 'type': mimeType,
+ 'mime': mimeType,
+ 'container': 'webm',
+ 'src': [dash_urls, audio_urls],
+ 'size': int(dash_size) + int(audio_size)
+ }

def extract(self, **kwargs):
if not self.streams_sorted:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/extractors/zhibo.py
new/you-get-0.4.1295/src/you_get/extractors/zhibo.py
--- old/you-get-0.4.1270/src/you_get/extractors/zhibo.py 2019-03-08
23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/src/you_get/extractors/zhibo.py 2019-04-25
16:23:50.000000000 +0200
@@ -14,7 +14,7 @@

video_html = r1(r'<script
type="text/javascript">([\s\S]*)</script></head>', html)

- # video_guessulike = r1(r"window.xgData
=([s\S'\s\.]*)\'\;[\s\S]*window.vouchData", video_html)
+ # video_guessulike = r1(r"window.xgData
=([s\S'\s\.]*)\'\;[\s\S]*window.vouchData", video_html)
video_url = r1(r"window.vurl = \'([s\S'\s\.]*)\'\;[\s\S]*window.imgurl",
video_html)
part_urls.append(video_url)
ext = video_url.split('.')[-1]
@@ -34,7 +34,7 @@
html = get_html(url)
title = r1(r'<title>([\s\S]*)</title>', html)
is_live =
r1(r"window.videoIsLive=\'([s\S'\s\.]*)\'\;[\s\S]*window.resDomain", html)
- if is_live is not "1":
+ if is_live != "1":
raise ValueError("The live stream is not online! (Errno:%s)" % is_live)

match = re.search(r"""
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/src/you_get/version.py
new/you-get-0.4.1295/src/you_get/version.py
--- old/you-get-0.4.1270/src/you_get/version.py 2019-03-08 23:20:06.000000000
+0100
+++ new/you-get-0.4.1295/src/you_get/version.py 2019-04-25 16:23:50.000000000
+0200
@@ -1,4 +1,4 @@
#!/usr/bin/env python

script_name = 'you-get'
-__version__ = '0.4.1270'
+__version__ = '0.4.1295'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/you-get-0.4.1270/tests/test.py
new/you-get-0.4.1295/tests/test.py
--- old/you-get-0.4.1270/tests/test.py 2019-03-08 23:20:06.000000000 +0100
+++ new/you-get-0.4.1295/tests/test.py 2019-04-25 16:23:50.000000000 +0200
@@ -7,6 +7,7 @@
magisto,
youtube,
bilibili,
+ toutiao,
)


@@ -31,5 +32,9 @@
info_only=True
)

+ def test_toutiao(self):
+
toutiao.download('https://www.365yg.com/i6640053613567675662/#mid=1611922564114440',
info_only=True)
+
+
if __name__ == '__main__':
unittest.main()


< Previous Next >
This Thread
  • No further messages