Hello community, here is the log from the commit of package python-cfscrape for openSUSE:Factory checked in at 2018-04-05 15:34:44 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-cfscrape (Old) and /work/SRC/openSUSE:Factory/.python-cfscrape.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-cfscrape" Thu Apr 5 15:34:44 2018 rev:2 rq:593651 version:1.9.5 Changes: -------- --- /work/SRC/openSUSE:Factory/python-cfscrape/python-cfscrape.changes 2018-02-14 10:51:08.546719399 +0100 +++ /work/SRC/openSUSE:Factory/.python-cfscrape.new/python-cfscrape.changes 2018-04-05 15:34:49.633246285 +0200 @@ -1,0 +2,6 @@ +Thu Apr 5 03:59:57 UTC 2018 - 9+suse@cirno.systems + +- Update to 1.9.5 + * Parse updated IUAM Javascript challenge + +------------------------------------------------------------------- Old: ---- cfscrape-1.9.4.tar.gz New: ---- cfscrape-1.9.5.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-cfscrape.spec ++++++ --- /var/tmp/diff_new_pack.qsvgET/_old 2018-04-05 15:34:50.313221710 +0200 +++ /var/tmp/diff_new_pack.qsvgET/_new 2018-04-05 15:34:50.313221710 +0200 @@ -18,7 +18,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-cfscrape -Version: 1.9.4 +Version: 1.9.5 Release: 0 Summary: Python module to bypass Cloudflare's anti-bot page License: MIT ++++++ cfscrape-1.9.4.tar.gz -> cfscrape-1.9.5.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cfscrape-1.9.4/PKG-INFO new/cfscrape-1.9.5/PKG-INFO --- old/cfscrape-1.9.4/PKG-INFO 2018-02-05 20:55:50.000000000 +0100 +++ new/cfscrape-1.9.5/PKG-INFO 2018-04-05 03:21:15.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cfscrape -Version: 1.9.4 +Version: 1.9.5 Summary: A simple Python module to bypass Cloudflare's anti-bot page. See https://github.com/Anorov/cloudflare-scrape for more information. Home-page: https://github.com/Anorov/cloudflare-scrape Author: Anorov diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cfscrape-1.9.4/README.md new/cfscrape-1.9.5/README.md --- old/cfscrape-1.9.4/README.md 2018-02-05 19:55:57.000000000 +0100 +++ new/cfscrape-1.9.5/README.md 2018-04-04 21:54:02.000000000 +0200 @@ -31,8 +31,7 @@ * Python 2.6 - 3.x * **[Requests](https://github.com/kennethreitz/requests)** >= 2.0 -* **[PyExecJS](https://pypi.python.org/pypi/PyExecJS)** -* **Node.js** is required for (safe) Javascript execution. +* **[Node.js](https://nodejs.org/)** * Your computer or server may already have it (check with `node -v`). If not, you can install it with `apt-get install nodejs` on Ubuntu and Debian. Otherwise, please read [Node's installation instructions](https://nodejs.org/en/download/package-manager/). `python setup.py install` will install the Python dependencies automatically. Node is the only application you need to install yourself. @@ -87,7 +86,7 @@ Normally, when a browser is faced with a Cloudflare IUAM challenge page, Cloudflare requires the browser to wait 5 seconds before submitting the challenge answer. If a website is under heavy load, sometimes this may fail. One solution is to increase the delay (perhaps to 10 or 15 seconds, depending on the website). If you would like to override this delay, pass the `delay` keyword argument to `create_scraper()` or `CloudflareScraper()`. -There is no need to override this delay unless cloudflare-scrape is generates an error recommending you increase the delay. +There is no need to override this delay unless cloudflare-scrape generates an error recommending you increase the delay. ```python scraper = cfscrape.create_scraper(delay=10) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cfscrape-1.9.4/cfscrape/__init__.py new/cfscrape-1.9.5/cfscrape/__init__.py --- old/cfscrape-1.9.4/cfscrape/__init__.py 2018-02-05 20:05:55.000000000 +0100 +++ new/cfscrape-1.9.5/cfscrape/__init__.py 2018-04-05 03:18:45.000000000 +0200 @@ -12,14 +12,16 @@ except ImportError: from urllib.parse import urlparse -__version__ = "1.9.4" +__version__ = "1.9.5" DEFAULT_USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0" + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/65.0.3325.181 Chrome/65.0.3325.181 Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; Moto G (5) Build/NPPS25.137-93-8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.137 Mobile Safari/537.36", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_4 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B554a Safari/9537.53", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:59.0) Gecko/20100101 Firefox/59.0", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0" ] DEFAULT_USER_AGENT = random.choice(DEFAULT_USER_AGENTS) @@ -34,8 +36,8 @@ ANSWER_ACCEPT_ERROR = """\ The challenge answer was not properly accepted by Cloudflare. This can occur if \ the target website is under heavy load, or if Cloudflare is experiencing issues. You can -potentially resolve this by increasing the challenge answer delay (default: 5 seconds). \ -For example: cfscrape.create_scraper(delay=10) +potentially resolve this by increasing the challenge answer delay (default: 8 seconds). \ +For example: cfscrape.create_scraper(delay=15) If increasing the delay does not help, please open a GitHub issue at \ https://github.com/Anorov/cloudflare-scrape/issues\ @@ -43,7 +45,7 @@ class CloudflareScraper(Session): def __init__(self, *args, **kwargs): - self.delay = kwargs.pop("delay", 5) + self.delay = kwargs.pop("delay", 8) super(CloudflareScraper, self).__init__(*args, **kwargs) if "requests" in self.headers["User-Agent"]: @@ -64,8 +66,6 @@ # Check if Cloudflare anti-bot is on if self.is_cloudflare_challenge(resp): resp = self.solve_cf_challenge(resp, **kwargs) - if self.is_cloudflare_challenge(resp): - raise ValueError(ANSWER_ACCEPT_ERROR) return resp @@ -94,7 +94,7 @@ raise ValueError("Unable to parse Cloudflare anti-bots page: %s %s" % (e.message, BUG_REPORT)) # Solve the Javascript challenge - params["jschl_answer"] = str(self.solve_challenge(body) + len(domain)) + params["jschl_answer"] = self.solve_challenge(body, domain) # Requests transforms any request into a GET after a redirect, # so the redirect has to be handled manually here to allow for @@ -109,21 +109,21 @@ return self.request(method, redirect_url, **original_kwargs) return self.request(method, redirect.headers["Location"], **original_kwargs) - def solve_challenge(self, body): + def solve_challenge(self, body, domain): try: js = re.search(r"setTimeout\(function\(\){\s+(var " "s,t,o,p,b,r,e,a,k,i,n,g,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n", body).group(1) except Exception: raise ValueError("Unable to identify Cloudflare IUAM Javascript on website. %s" % BUG_REPORT) - js = re.sub(r"a\.value = (parseInt\(.+?\)).+", r"\1", js) - js = re.sub(r"\s{3,}[a-z](?: = |\.).+", "", js) + js = re.sub(r"a\.value = (.+ \+ t\.length).+", r"\1", js) + js = re.sub(r"\s{3,}[a-z](?: = |\.).+", "", js).replace("t.length", str(len(domain))) # Strip characters that could be used to exit the string context # These characters are not currently used in Cloudflare's arithmetic snippet js = re.sub(r"[\n\\']", "", js) - if "parseInt" not in js: + if "toFixed" not in js: raise ValueError("Error parsing Cloudflare IUAM Javascript challenge. %s" % BUG_REPORT) # Use vm.runInNewContext to safely evaluate code @@ -134,7 +134,7 @@ result = subprocess.check_output(["node", "-e", js]).strip() except OSError as e: if e.errno == 2: - raise EnvironmentError("Missing Node.js runtime. Node is required. Please read the cfscrape" + raise EnvironmentError("Missing Node.js runtime. Node is required and must be in the PATH (check with `node -v`). Your Node binary may be called `nodejs` rather than `node`, in which case you may need to run `apt-get install nodejs-legacy` on some Debian-based systems. (Please read the cfscrape" " README's Dependencies section: https://github.com/Anorov/cloudflare-scrape#dependencies.") raise except Exception: @@ -142,7 +142,7 @@ raise try: - result = int(result) + float(result) except Exception: raise ValueError("Cloudflare IUAM challenge returned unexpected answer. %s" % BUG_REPORT) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cfscrape-1.9.4/cfscrape.egg-info/PKG-INFO new/cfscrape-1.9.5/cfscrape.egg-info/PKG-INFO --- old/cfscrape-1.9.4/cfscrape.egg-info/PKG-INFO 2018-02-05 20:55:49.000000000 +0100 +++ new/cfscrape-1.9.5/cfscrape.egg-info/PKG-INFO 2018-04-05 03:21:15.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cfscrape -Version: 1.9.4 +Version: 1.9.5 Summary: A simple Python module to bypass Cloudflare's anti-bot page. See https://github.com/Anorov/cloudflare-scrape for more information. Home-page: https://github.com/Anorov/cloudflare-scrape Author: Anorov diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cfscrape-1.9.4/cfscrape.egg-info/requires.txt new/cfscrape-1.9.5/cfscrape.egg-info/requires.txt --- old/cfscrape-1.9.4/cfscrape.egg-info/requires.txt 2018-02-05 20:55:49.000000000 +0100 +++ new/cfscrape-1.9.5/cfscrape.egg-info/requires.txt 2018-04-05 03:21:15.000000000 +0200 @@ -1,2 +1 @@ -PyExecJS >= 1.4.0 requests >= 2.0.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cfscrape-1.9.4/setup.cfg new/cfscrape-1.9.5/setup.cfg --- old/cfscrape-1.9.4/setup.cfg 2018-02-05 20:55:50.000000000 +0100 +++ new/cfscrape-1.9.5/setup.cfg 2018-04-05 03:21:15.000000000 +0200 @@ -1,5 +1,5 @@ [egg_info] -tag_svn_revision = 0 tag_date = 0 +tag_svn_revision = 0 tag_build = diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cfscrape-1.9.4/setup.py new/cfscrape-1.9.5/setup.py --- old/cfscrape-1.9.4/setup.py 2018-02-05 19:59:30.000000000 +0100 +++ new/cfscrape-1.9.5/setup.py 2018-04-04 21:51:19.000000000 +0200 @@ -18,5 +18,5 @@ url = 'https://github.com/Anorov/cloudflare-scrape', keywords = ['cloudflare', 'scraping'], include_package_data = True, - install_requires = ['PyExecJS >= 1.4.0', 'requests >= 2.0.0'] + install_requires = ['requests >= 2.0.0'] )