Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package you-get for openSUSE:Factory checked in at 2024-06-24 20:56:26
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/you-get (Old)
and /work/SRC/openSUSE:Factory/.you-get.new.18349 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "you-get"
Mon Jun 24 20:56:26 2024 rev:47 rq:1182973 version:0.4.1710
Changes:
--------
--- /work/SRC/openSUSE:Factory/you-get/you-get.changes 2024-05-22 21:32:29.650016808 +0200
+++ /work/SRC/openSUSE:Factory/.you-get.new.18349/you-get.changes 2024-06-24 20:57:56.842489257 +0200
@@ -1,0 +2,6 @@
+Mon Jun 24 08:08:54 UTC 2024 - Luigi Baldoni
+
+- Update to version 0.4.1710
+ * YouTube: Fix 403 error and throttling
+
+-------------------------------------------------------------------
Old:
----
you-get-0.4.1700.tar.gz
New:
----
you-get-0.4.1710.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ you-get.spec ++++++
--- /var/tmp/diff_new_pack.moGdY6/_old 2024-06-24 20:57:57.230503435 +0200
+++ /var/tmp/diff_new_pack.moGdY6/_new 2024-06-24 20:57:57.234503582 +0200
@@ -17,7 +17,7 @@
Name: you-get
-Version: 0.4.1700
+Version: 0.4.1710
Release: 0
Summary: Dumb downloader that scrapes the web
License: MIT
++++++ you-get-0.4.1700.tar.gz -> you-get-0.4.1710.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1700/.github/workflows/python-package.yml new/you-get-0.4.1710/.github/workflows/python-package.yml
--- old/you-get-0.4.1700/.github/workflows/python-package.yml 2024-05-22 01:58:47.000000000 +0200
+++ new/you-get-0.4.1710/.github/workflows/python-package.yml 2024-06-23 21:04:52.000000000 +0200
@@ -26,7 +26,7 @@
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools
- pip install flake8 pytest
+ pip install flake8
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- name: Lint with flake8
run: |
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1700/.gitignore new/you-get-0.4.1710/.gitignore
--- old/you-get-0.4.1700/.gitignore 2024-05-22 01:58:47.000000000 +0200
+++ new/you-get-0.4.1710/.gitignore 2024-06-23 21:04:52.000000000 +0200
@@ -79,6 +79,7 @@
*.ts
*.webm
*.xml
+*.json
/.env
/.idea
*.m4a
@@ -88,5 +89,5 @@
*.zip
+.emacs*
.vscode
-
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1700/Makefile new/you-get-0.4.1710/Makefile
--- old/you-get-0.4.1700/Makefile 2024-05-22 01:58:47.000000000 +0200
+++ new/you-get-0.4.1710/Makefile 2024-06-23 21:04:52.000000000 +0200
@@ -8,7 +8,7 @@
@(cd src/; python3 -i -c 'import you_get; print("You-Get %s\n>>> import you_get" % you_get.version.__version__)')
test:
- $(SETUP) test
+ (cd src; python -m unittest discover -s ../tests)
clean:
zenity --question
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1700/requirements.txt new/you-get-0.4.1710/requirements.txt
--- old/you-get-0.4.1700/requirements.txt 1970-01-01 01:00:00.000000000 +0100
+++ new/you-get-0.4.1710/requirements.txt 2024-06-23 21:04:52.000000000 +0200
@@ -0,0 +1,2 @@
+# runtime dependencies
+dukpy
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1700/setup.py new/you-get-0.4.1710/setup.py
--- old/you-get-0.4.1700/setup.py 2024-05-22 01:58:47.000000000 +0200
+++ new/you-get-0.4.1710/setup.py 2024-06-23 21:04:52.000000000 +0200
@@ -56,7 +56,8 @@
entry_points = {'console_scripts': proj_info['console_scripts']},
- extras_require={
+ install_requires = ['dukpy'],
+ extras_require = {
'socks': ['PySocks'],
}
)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1700/src/you_get/extractors/youtube.py new/you-get-0.4.1710/src/you_get/extractors/youtube.py
--- old/you-get-0.4.1700/src/you_get/extractors/youtube.py 2024-05-22 01:58:47.000000000 +0200
+++ new/you-get-0.4.1710/src/you_get/extractors/youtube.py 2024-06-23 21:04:52.000000000 +0200
@@ -3,6 +3,13 @@
from ..common import *
from ..extractor import VideoExtractor
+try:
+ import dukpy
+except ImportError:
+ log.e('Please install dukpy in order to extract videos from YouTube:')
+ log.e('$ pip install dukpy')
+ exit(0)
+from urllib.parse import urlparse, parse_qs, urlencode
from xml.dom.minidom import parseString
class YouTube(VideoExtractor):
@@ -68,45 +75,32 @@
'audio_encoding': 'AAC', 'audio_bitrate': '24'},
]
+ def dethrottle(js, url):
+ def n_to_n(js, n):
+ # Examples:
+ # yma - https://www.youtube.com/s/player/84314bef/player_ias.vflset/en_US/base.js
+ # Xka - https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/sv_SE/base.js
+ f1 = match1(js, r'a\.set\("n",b\),[$\w]+\.length\|\|([$\w]+)\(""\)')
+ f1def = match1(js, r'\W%s=(function\(\w+\).+?\)});' % re.escape(f1))
+ n = dukpy.evaljs('(%s)("%s")' % (f1def, n))
+ return n
+
+ u = urlparse(url)
+ qs = parse_qs(u.query)
+ n = n_to_n(js, qs['n'][0])
+ qs['n'] = [n]
+ return u._replace(query=urlencode(qs, doseq=True)).geturl()
+
def s_to_sig(js, s):
# Examples:
- # - https://www.youtube.com/yts/jsbin/player-da_DK-vflWlK-zq/base.js
- # - https://www.youtube.com/yts/jsbin/player-vflvABTsY/da_DK/base.js
- # - https://www.youtube.com/yts/jsbin/player-vfls4aurX/da_DK/base.js
- # - https://www.youtube.com/yts/jsbin/player_ias-vfl_RGK2l/en_US/base.js
- # - https://www.youtube.com/yts/jsbin/player-vflRjqq_w/da_DK/base.js
- # - https://www.youtube.com/yts/jsbin/player_ias-vfl-jbnrr/da_DK/base.js
- # - https://www.youtube.com/s/player/0b643cd1/player_ias.vflset/sv_SE/base.js
- # - https://www.youtube.com/s/player/50e823fc/player_ias.vflset/sv_SE/base.js
- # - https://www.youtube.com/s/player/3b5d5649/player_ias.vflset/sv_SE/base.js
- # - https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/sv_SE/base.js
- def tr_js(code):
- code = re.sub(r'function', r'def', code)
- # add prefix '_sig_' to prevent namespace pollution
- code = re.sub(r'(\W)([$\w][$\w][$\w]?)\(', r'\1_sig_\2(', code)
- code = re.sub(r'\$', '_dollar', code)
- code = re.sub(r'\{', r': ', code)
- code = re.sub(r'\}', r'\n', code)
- code = re.sub(r'var\s+', r'', code)
- code = re.sub(r'(\w+).join\(""\)', r'"".join(\1)', code)
- code = re.sub(r'(\w+).length', r'len(\1)', code)
- code = re.sub(r'(\w+).slice\((\w+)\)', r'\1[\2:]', code)
- code = re.sub(r'(\w+).splice\((\w+),(\w+)\)', r'del \1[\2:\2+\3]', code)
- code = re.sub(r'(\w+).split\(""\)', r'list(\1)', code)
- return code
-
- js = js.replace('\n', ' ')
- f1 = match1(js, r'\.set\(\w+\.sp,encodeURIComponent\(([$\w]+)') or \
- match1(js, r'\.set\(\w+\.sp,\(0,window\.encodeURIComponent\)\(([$\w]+)') or \
- match1(js, r'\.set\(\w+\.sp,([$\w]+)\(\w+\.s\)\)') or \
- match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)') or \
- match1(js, r'=([$\w]+)\(decodeURIComponent\(')
- f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
- match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
- f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)
+ # BPa - https://www.youtube.com/s/player/84314bef/player_ias.vflset/en_US/base.js
+ # Xva - https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/sv_SE/base.js
+ js_code = ''
+ f1 = match1(js, r'=([$\w]+)\(decodeURIComponent\(')
+ f1def = match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
+ f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def) # remove . prefix
f1def = 'function %s%s' % (f1, f1def)
- code = tr_js(f1def)
- f2s = set(re.findall(r'([$\w]+)\(\w+,\d+\)', f1def))
+ f2s = set(re.findall(r'([$\w]+)\(\w+,\d+\)', f1def)) # find all invoked function names
for f2 in f2s:
f2e = re.escape(f2)
f2def = re.search(r'[^$\w]%s:function\((\w+,\w+)\)(\{[^\{\}]+\})' % f2e, js)
@@ -115,13 +109,10 @@
else:
f2def = re.search(r'[^$\w]%s:function\((\w+)\)(\{[^\{\}]+\})' % f2e, js)
f2def = 'function {}({},b){}'.format(f2e, f2def.group(1), f2def.group(2))
- f2 = re.sub(r'\$', '_dollar', f2) # replace dollar sign
- code = code + 'global _sig_%s\n' % f2 + tr_js(f2def)
-
- f1 = re.sub(r'\$', '_dollar', f1) # replace dollar sign
- code = code + '_sig=_sig_%s(s)' % f1
- exec(code, globals(), locals())
- return locals()['_sig']
+ js_code += f2def + ';'
+ js_code += f1def + ';%s("%s")' % (f1, s)
+ sig = dukpy.evaljs(js_code)
+ return sig
def chunk_by_range(url, size):
urls = []
@@ -196,188 +187,67 @@
if re.search('\Wlist=', self.url) and not kwargs.get('playlist'):
log.w('This video is from a playlist. (use --playlist to download all videos in the playlist.)')
- # Get video info
- # 'eurl' is a magic parameter that can bypass age restriction
- # full form: 'eurl=https%3A%2F%2Fyoutube.googleapis.com%2Fv%2F{VIDEO_ID}'
- #video_info = parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}&eurl=https%3A%2F%2Fy'.format(self.vid)))
- #logging.debug('STATUS: %s' % video_info['status'][0])
- video_info = {'status': ['ok'], 'use_cipher_signature': 'True'}
-
- ytplayer_config = None
- if 'status' not in video_info:
- log.wtf('[Failed] Unknown status.', exit_code=None)
- raise
- elif video_info['status'] == ['ok']:
- if 'use_cipher_signature' not in video_info or video_info['use_cipher_signature'] == ['False']:
- self.title = parse.unquote_plus(json.loads(video_info["player_response"][0])["videoDetails"]["title"])
- # Parse video page (for DASH)
- video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
- try:
- try:
- # Complete ytplayer_config
- ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
-
- # Workaround: get_video_info returns bad s. Why?
- if 'url_encoded_fmt_stream_map' not in ytplayer_config['args']:
- stream_list = json.loads(ytplayer_config['args']['player_response'])['streamingData']['formats']
- else:
- stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
- #stream_list = ytplayer_config['args']['adaptive_fmts'].split(',')
-
- if 'assets' in ytplayer_config:
- self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
- elif re.search('([^"]*/base\.js)"', video_page):
- self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
- self.html5player = self.html5player.replace('\/', '/') # unescape URL
- else:
- self.html5player = None
-
- except:
- # ytplayer_config = {args:{raw_player_response:ytInitialPlayerResponse}}
- try: # FIXME: we should extract ytInitialPlayerResponse more reliably
- ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});</script>', video_page).group(1))
- except:
- ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
-
- stream_list = ytInitialPlayerResponse['streamingData']['formats']
- #stream_list = ytInitialPlayerResponse['streamingData']['adaptiveFormats']
-
- if re.search('([^"]*/base\.js)"', video_page):
- self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
- else:
- self.html5player = None
-
- except:
- if 'url_encoded_fmt_stream_map' not in video_info:
- stream_list = json.loads(video_info['player_response'][0])['streamingData']['formats']
- else:
- stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
-
- if re.search('([^"]*/base\.js)"', video_page):
- self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
- else:
- self.html5player = None
-
- else:
- # Parse video page instead
- video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
-
- try: # FIXME: we should extract ytInitialPlayerResponse more reliably
- ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});</script>', video_page).group(1))
- except:
- ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
-
- self.title = ytInitialPlayerResponse["videoDetails"]["title"]
- if re.search('([^"]*/base\.js)"', video_page):
- self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
- else:
- self.html5player = None
-
- stream_list = ytInitialPlayerResponse['streamingData']['formats']
+ # Extract from video page
+ logging.debug('Extracting from the video page...')
+ video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
- elif video_info['status'] == ['fail']:
- logging.debug('ERRORCODE: %s' % video_info['errorcode'][0])
- if video_info['errorcode'] == ['150']:
- # FIXME: still relevant?
- if cookies:
- # Load necessary cookies into headers (for age-restricted videos)
- consent, ssid, hsid, sid = 'YES', '', '', ''
- for cookie in cookies:
- if cookie.domain.endswith('.youtube.com'):
- if cookie.name == 'SSID':
- ssid = cookie.value
- elif cookie.name == 'HSID':
- hsid = cookie.value
- elif cookie.name == 'SID':
- sid = cookie.value
- cookie_str = 'CONSENT=%s; SSID=%s; HSID=%s; SID=%s' % (consent, ssid, hsid, sid)
-
- video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid,
- headers={'Cookie': cookie_str})
- else:
- video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
-
- try:
- ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+});ytplayer', video_page).group(1))
- except:
- msg = re.search('class="message">([^<]+)<', video_page).group(1)
- log.wtf('[Failed] Got message "%s". Try to login with --cookies.' % msg.strip())
-
- if 'title' in ytplayer_config['args']:
- # 150 Restricted from playback on certain sites
- # Parse video page instead
- self.title = ytplayer_config['args']['title']
- self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
- stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
- else:
- log.wtf('[Error] The uploader has not made this video available in your country.', exit_code=None)
- raise
- #self.title = re.search('http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3...', # noqa
# info_only=True
#)
- #youtube.download(
- # 'https://www.youtube.com/watch?v=Fpr4fQSh1cc', info_only=True
- #)
+ youtube.download(
+ 'https://www.youtube.com/watch?v=oRdxUFDoQe0', info_only=True
+ )
def test_acfun(self):
acfun.download('https://www.acfun.cn/v/ac44560432', info_only=True)
- def test_bilibili(self):
- bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
+ #def test_bilibili(self):
+ #bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
#def test_soundcloud(self):
## single song