Hello community, here is the log from the commit of package python-fanficfare for openSUSE:Factory checked in at 2019-05-03 22:47:26 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-fanficfare (Old) and /work/SRC/openSUSE:Factory/.python-fanficfare.new.5148 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-fanficfare" Fri May 3 22:47:26 2019 rev:8 rq:700318 version:3.7.6 Changes: -------- --- /work/SRC/openSUSE:Factory/python-fanficfare/python-fanficfare.changes 2019-04-28 20:15:41.750334864 +0200 +++ /work/SRC/openSUSE:Factory/.python-fanficfare.new.5148/python-fanficfare.changes 2019-05-03 22:47:26.915909991 +0200 @@ -1,0 +2,15 @@ +Fri May 3 08:45:55 CEST 2019 - Matej Cepl <mcepl@suse.com> + +- Update to 3.7.6: + - Fix CLI Upload to have correct version. + - Fix for BS halping with string conversions on PI update from + Saved Meta Column. + - More improvements for adapter_asianfanficscom, including + auto_sub feature, thanks oh45454545 + - Improvements for adapter_asianfanficscom, thanks oh45454545 + - Update adapter_asianfanficscom to fetch chapter texts from + JSON url. + - Ad wall indicator has changed for adapter_webnovelcom. + - Fix use_archived_author in AO3. + +------------------------------------------------------------------- Old: ---- FanFicFare-3.7.0.tar.gz New: ---- FanFicFare-3.7.6.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-fanficfare.spec ++++++ --- /var/tmp/diff_new_pack.slTqOD/_old 2019-05-03 22:47:27.411911130 +0200 +++ /var/tmp/diff_new_pack.slTqOD/_new 2019-05-03 22:47:27.415911139 +0200 @@ -20,7 +20,7 @@ %define modnamedown fanficfare %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-fanficfare -Version: 3.7.0 +Version: 3.7.6 Release: 0 Summary: Tool for making eBooks from stories on fanfiction and other web sites License: GPL-3.0-only ++++++ FanFicFare-3.7.0.tar.gz -> FanFicFare-3.7.6.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/calibre-plugin/__init__.py new/FanFicFare-3.7.6/calibre-plugin/__init__.py --- old/FanFicFare-3.7.0/calibre-plugin/__init__.py 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/calibre-plugin/__init__.py 2019-04-24 05:29:17.000000000 +0200 @@ -33,7 +33,7 @@ from calibre.customize import InterfaceActionBase # pulled out from FanFicFareBase for saving in prefs.py -__version__ = (3, 7, 0) +__version__ = (3, 7, 6) ## Apparently the name for this class doesn't matter--it was still ## 'demo' for the first few versions. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/calibre-plugin/plugin-defaults.ini new/FanFicFare-3.7.6/calibre-plugin/plugin-defaults.ini --- old/FanFicFare-3.7.0/calibre-plugin/plugin-defaults.ini 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/calibre-plugin/plugin-defaults.ini 2019-04-24 05:29:17.000000000 +0200 @@ -2133,10 +2133,11 @@ [www.asianfanfics.com] ## Unlike most sites, asianfanfics.com, instead of denying access to -## 'adult' content, will censor the text of stories to remove the -## 'adult' words. FanFicFare cannot detect when this happens, but if -## you set your username and password, FFF will log you in to prevent -## that. +## 'adult' or subscriber-only content, will censor the text of stories +## to remove 'adult' words or entire portions of the text. This is why +## an account is required to download stories from this site. It is +## also strongly recommended to consider enabling auto_sub in order to +## further avoid this from happening. #username:YourName #password:yourpassword @@ -2163,6 +2164,14 @@ ## chapter_start to remove the standard chapter title, as shown below. inject_chapter_title:false +## This website removes certain HTML tags and portions of the story +## from subscriber-only stories. It is strongly recommended to turn +## this option on. This will automatically subscribe you to such +## stories in order to acquire the unaltered text. You can +## unsubscribe manually on the website after the story has been +## downloaded. +auto_sub:false + [www.bdsmlibrary.com] ## Some sites also require the user to confirm they are adult for ## adult content. Uncomment by removing '#' in front of is_adult. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/fanficfare/adapters/adapter_archiveofourownorg.py new/FanFicFare-3.7.6/fanficfare/adapters/adapter_archiveofourownorg.py --- old/FanFicFare-3.7.0/fanficfare/adapters/adapter_archiveofourownorg.py 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/fanficfare/adapters/adapter_archiveofourownorg.py 2019-04-24 05:29:17.000000000 +0200 @@ -225,8 +225,7 @@ # Hope Roy [archived by <a href="/users/ssa_archivist/pseuds/ssa_archivist" rel="author">ssa_archivist</a>] # </h3> # stripped:"Hope Roy [archived by ssa_archivist]" - - m = re.match(r'(?P<author>.*) \[archived by (?P<archivist>.*)\]',stripHTML(byline)) + m = re.match(r'(?P<author>.*) \[archived by ?(?P<archivist>.*)\]',stripHTML(byline)) if( m and len(alist) == 1 and self.getConfig('use_archived_author') ): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/fanficfare/adapters/adapter_asianfanficscom.py new/FanFicFare-3.7.6/fanficfare/adapters/adapter_asianfanficscom.py --- old/FanFicFare-3.7.0/fanficfare/adapters/adapter_asianfanficscom.py 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/fanficfare/adapters/adapter_asianfanficscom.py 2019-04-24 05:29:17.000000000 +0200 @@ -4,6 +4,7 @@ import logging logger = logging.getLogger(__name__) import re +import json from ..htmlcleanup import stripHTML from .. import exceptions as exceptions @@ -70,45 +71,40 @@ else: params['username'] = self.getConfig("username") params['password'] = self.getConfig("password") + + if not params['username']: + raise exceptions.FailedToLogin(url,params['username']) + params['from_url'] = url params['csrf_aff_token'] = soup.find('input',{'name':'csrf_aff_token'})['value'] + if not params['csrf_aff_token']: + raise exceptions.FailedToDownload('Error when logging in. This usually means a change in the website code.') + loginUrl = 'https://' + self.getSiteDomain() + '/login/index' logger.info("Will now login to URL (%s) as (%s)" % (loginUrl, params['username'])) - d = self._postUrl(loginUrl, params, usecache=False) - - if params['username'] not in d: # check if username is mentioned in output (logged in as, var visitorName, etc.) - logger.info("Failed to login to URL %s as %s" % (loginUrl, params['username'])) + data = self._postUrl(loginUrl, params) + soup = self.make_soup(data) + if self.loginNeededCheck(soup): + logger.info('Failed to login to URL %s as %s' % (loginUrl, params['username'])) raise exceptions.FailedToLogin(url,params['username']) - return False - else: - return True - def doAdultCheck(self, url, soup): - check = soup.find('form',{'action':'/account/toggle_age'}) - if check: - logger.debug("Found adult check") - if self.is_adult or self.getConfig("is_adult"): - contentFilter = check.find('a',{'href':'/account/mark_over_18'}) #two different types of adult checks - if contentFilter: - loginUrl = 'https://' + self.getSiteDomain() + '/account/mark_over_18' - self._fetchUrl(loginUrl) - else: - params = {} - params['csrf_aff_token'] = check.find('input',{'name':'csrf_aff_token'})['value'] - params['is_of_age'] = '1' - params['current_url'] = '/story/view/' + self.story.getMetadata('storyId') - loginUrl = 'https://' + self.getSiteDomain() + '/account/toggle_age' - self._postUrl(loginUrl,params) - - data = self._fetchUrl(url,usecache=False) - soup = self.make_soup(data) - if "Are you over 18 years old" in data: - raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) - else: - return soup + def loginNeededCheck(self,soup): + return soup.find('div',{'id':'login'}) != None + + def doStorySubscribe(self, url, soup): + subHref = soup.find('a',{'id':'subscribe'}) + if subHref: + #does not work when using https - 403 + subUrl = 'http://' + self.getSiteDomain() + subHref['href'] + self._fetchUrl(subUrl) + data = self._fetchUrl(url,usecache=False) + soup = self.make_soup(data) + check = soup.find('div',{'class':'click-to-read-full'}) + if check: + return False else: - raise exceptions.AdultCheckRequired(self.url) + return soup else: return False @@ -123,7 +119,6 @@ def doExtractChapterUrlsAndMetadata(self,get_cover=True): url = self.url logger.info("url: "+url) - try: data = self._fetchUrl(url) @@ -136,18 +131,23 @@ # use BeautifulSoup HTML parser to make everything easier to find. soup = self.make_soup(data) - # it is best to log in whenever possible, unless already logged in from cache.. - if self.password or self.getConfig("password") and "Logout" not in data: + if self.loginNeededCheck(soup): + # always login if not already to avoid lots of headaches self.performLogin(url,soup) + # refresh website after logging in data = self._fetchUrl(url,usecache=False) soup = self.make_soup(data) - elif "Logout" not in data: - logger.info('Note: Logging in is highly recommended, as this website censors text if not logged in.') - # adult check - self.checkSoup = self.doAdultCheck(url,soup) - if self.checkSoup: - soup = self.checkSoup + # subscription check + subCheck = soup.find('div',{'class':'click-to-read-full'}) + if subCheck and self.getConfig("auto_sub"): + subSoup = self.doStorySubscribe(url,soup) + if subSoup: + soup = subSoup + else: + raise exceptions.FailedToDownload("Error when subscribing to story. This usually means a change in the website code.") + elif subCheck and not self.getConfig("auto_sub"): + raise exceptions.FailedToDownload("This story is only available to subscribers. You can subscribe manually on the web site, or set auto_sub:true in personal.ini.") ## Title a = soup.find('h1', {'id': 'story-title'}) @@ -171,6 +171,7 @@ for index, chapter in enumerate(chapters): if chapter.text != 'Foreword': # skip the foreword self.add_chapter(chapter.text,'https://' + self.getSiteDomain() + chapter['value']) # note: AFF cuts off chapter names in list. this gets kind of fixed later on + # find timestamp a = soup.find('span', text='Updated') if a == None: @@ -189,7 +190,10 @@ self.story.setMetadata('status', 'In-Progress') # story description - a = soup.find('div', {'id':'story-description'}) + jsonlink = soup.find('link',href=re.compile(r'/api/forewords/[0-9]+/foreword_[0-9a-z]+.json')) + fore_json = json.loads(self._fetchUrl(jsonlink['href'])) + content = self.make_soup(fore_json['post']).find('body') # BS4 adds <html><body> if not present. + a = content.find('div', {'id':'story-description'}) if a: self.setDescription(url,a) @@ -216,6 +220,12 @@ a = a.parent.find('time') self.story.setMetadata('dateUpdated', makeDate(a['datetime'], self.dateformat)) + # word count + a = soup.find('span', text='Total Word Count') + if a: + a = a.find_next('span') + self.story.setMetadata('numWords', int(a.text.split()[0])) + # upvote, subs, and views a = soup.find('div',{'class':'title-meta'}) spans = a.findAll('span', recursive=False) @@ -241,22 +251,21 @@ data = self._fetchUrl(url) soup = self.make_soup(data) - # have to do adult check here as well because individual chapters can be marked as mature - if not self.checkSoup: - self.checkSoup = self.doAdultCheck(url,soup) - if self.checkSoup: - soup = self.checkSoup - - # grab contents - content = soup.find('div', {'id': 'user-submitted-body'}) - if content: + try: + # https://www.asianfanfics.com/api/chapters/4791923/chapter_46d32e413d1a702a26... + jsonlink = soup.find('link',href=re.compile(r'/api/chapters/[0-9]+/chapter_[0-9a-z]+.json')) + chap_json = json.loads(self._fetchUrl(jsonlink['href'])) + content = self.make_soup(chap_json['post']).find('body') # BS4 adds <html><body> if not present. + content.name='div' # change body to a div. if self.getConfig('inject_chapter_title'): + # the dumbest workaround ever for the abbreviated chapter titles from before logger.debug("Injecting full-length chapter title") newTitle = soup.find('h1', {'id' : 'chapter-title'}).text - newTitle = self.make_soup('<h3>%s</h3>' % (newTitle)) # the dumbest workaround ever for the abbreviated chapter titles from before + newTitle = self.make_soup('<h3>%s</h3>' % (newTitle)).find('body') # BS4 adds <html><body> if not present. + newTitle.name='div' # change body to a div. newTitle.append(content) return self.utf8FromSoup(url,newTitle) else: return self.utf8FromSoup(url,content) - else: - raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + except Exception as e: + raise exceptions.FailedToDownload("Error downloading Chapter: %s %s!" % (url,e)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/fanficfare/adapters/adapter_webnovelcom.py new/FanFicFare-3.7.6/fanficfare/adapters/adapter_webnovelcom.py --- old/FanFicFare-3.7.0/fanficfare/adapters/adapter_webnovelcom.py 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/fanficfare/adapters/adapter_webnovelcom.py 2019-04-24 05:29:17.000000000 +0200 @@ -179,11 +179,9 @@ for volume in jsondata["data"]["volumeItems"]: for chap in volume["chapterItems"]: # Only allow free and VIP type 1 chapters - if chap['isVip'] not in [0]: # removed VIP type 1 - # chapter (ad-wall'ed) - # because the ad-wall - # bypass code stopped - # working. --JM + if chap['isAuth'] not in [1]: # Ad wall indicator + # seems to have changed + # --JM continue chap_title = 'Chapter ' + unicode(chap['index']) + ' - ' + chap['name'] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/fanficfare/cli.py new/FanFicFare-3.7.6/fanficfare/cli.py --- old/FanFicFare-3.7.0/fanficfare/cli.py 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/fanficfare/cli.py 2019-04-24 05:29:17.000000000 +0200 @@ -39,7 +39,7 @@ def pickle_load(f): return pickle.load(f,encoding="bytes") -version="3.7.0" +version="3.7.6" os.environ['CURRENT_VERSION_ID']=version global_cache = 'global_cache' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/fanficfare/configurable.py new/FanFicFare-3.7.6/fanficfare/configurable.py --- old/FanFicFare-3.7.0/fanficfare/configurable.py 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/fanficfare/configurable.py 2019-04-24 05:29:17.000000000 +0200 @@ -244,6 +244,8 @@ 'inject_chapter_title':(['asianfanfics.com'],None,boollist), + 'auto_sub':(['asianfanfics.com'],None,boollist), + # eFiction Base adapters allow bulk_load # kept forgetting to add them, so now it's automatic. 'bulk_load':(adapters.get_bulk_load_sites(), @@ -441,6 +443,7 @@ 'conditionals_use_lists', 'description_in_chapter', 'inject_chapter_title', + 'auto_sub', 'titlepage_end', 'titlepage_entries', 'titlepage_entry', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/fanficfare/defaults.ini new/FanFicFare-3.7.6/fanficfare/defaults.ini --- old/FanFicFare-3.7.0/fanficfare/defaults.ini 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/fanficfare/defaults.ini 2019-04-24 05:29:17.000000000 +0200 @@ -2167,10 +2167,11 @@ [www.asianfanfics.com] ## Unlike most sites, asianfanfics.com, instead of denying access to -## 'adult' content, will censor the text of stories to remove the -## 'adult' words. FanFicFare cannot detect when this happens, but if -## you set your username and password, FFF will log you in to prevent -## that. +## 'adult' or subscriber-only content, will censor the text of stories +## to remove 'adult' words or entire portions of the text. This is why +## an account is required to download stories from this site. It is +## also strongly recommended to consider enabling auto_sub in order to +## further avoid this from happening. #username:YourName #password:yourpassword @@ -2197,6 +2198,14 @@ ## chapter_start to remove the standard chapter title, as shown below. inject_chapter_title:false +## This website removes certain HTML tags and portions of the story +## from subscriber-only stories. It is strongly recommended to turn +## this option on. This will automatically subscribe you to such +## stories in order to acquire the unaltered text. You can +## unsubscribe manually on the website after the story has been +## downloaded. +auto_sub:false + [www.bdsmlibrary.com] ## Some sites also require the user to confirm they are adult for ## adult content. Uncomment by removing '#' in front of is_adult. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/fanficfare/story.py new/FanFicFare-3.7.6/fanficfare/story.py --- old/FanFicFare-3.7.0/fanficfare/story.py 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/fanficfare/story.py 2019-04-24 05:29:17.000000000 +0200 @@ -703,7 +703,10 @@ elif 'list' in tag['class']: val = [] for i in tag.find_all('li'): - val.append(i.string) + # keeps & but removes <li></li> because BS4 + # halps by converting NavigableString to string + # (losing entities) + val.append(unicode(i)[4:-5]) elif 'int' in tag['class']: # Python reports true when asked isinstance(<bool>, (int)) # bools now converted to unicode when set. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/setup.py new/FanFicFare-3.7.6/setup.py --- old/FanFicFare-3.7.0/setup.py 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/setup.py 2019-04-24 05:29:17.000000000 +0200 @@ -27,7 +27,7 @@ name=package_name, # Versions should comply with PEP440. - version="3.7.0", + version="3.7.6", description='A tool for downloading fanfiction to eBook formats', long_description=long_description, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/webservice/app.yaml new/FanFicFare-3.7.6/webservice/app.yaml --- old/FanFicFare-3.7.0/webservice/app.yaml 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/webservice/app.yaml 2019-04-24 05:29:17.000000000 +0200 @@ -1,6 +1,6 @@ # ffd-retief-hrd fanficfare application: fanficfare -version: 3-7-0 +version: 3-7-6 runtime: python27 api_version: 1 threadsafe: true diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.7.0/webservice/index.html new/FanFicFare-3.7.6/webservice/index.html --- old/FanFicFare-3.7.0/webservice/index.html 2019-04-19 21:08:56.000000000 +0200 +++ new/FanFicFare-3.7.6/webservice/index.html 2019-04-24 05:29:17.000000000 +0200 @@ -84,7 +84,7 @@ If you have any problems with this application, please report them in the <a href="https://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>. The - <a href="https://3-6-0.fanficfare.appspot.com">previous version</a> + <a href="https://3-7-0.fanficfare.appspot.com">previous version</a> is also available for you to use if necessary. </p> <div id='error'>