Hello community, here is the log from the commit of package openSUSE-release-tools for openSUSE:Factory checked in at 2018-01-02 16:35:44 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old) and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "openSUSE-release-tools" Tue Jan 2 16:35:44 2018 rev:36 rq:561094 version:20180102.98a659c Changes: -------- --- /work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes 2017-12-29 18:51:04.442062391 +0100 +++ /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new/openSUSE-release-tools.changes 2018-01-02 16:35:50.539375711 +0100 @@ -1,0 +2,17 @@ +Tue Jan 02 10:50:33 UTC 2018 - opensuse-releaseteam@opensuse.org + +- Update to version 20180102.98a659c: + * manager42: ignore meta packages + * manager42: refactor + * manager42: don't turn dropped packages into fork + * manager42: warn about inconsistent packages + * reochecker: avoid accessing empty status + * repo_checker: only update comment if published or message changed. + * ReviewBot: comment_write(): provide info_extra_identical parameter. + * fcc_submitter: handle multibuild package in build_succeeded package list + * check_duplicate_binaries: better yaml + * pkglistgen: migrate bash scripts to python. + * osclib/stagingapi: provide is_staging_bootstrapped() from request_splitter. + * ToolBase: provide -d alias for --debug for consistency. + +------------------------------------------------------------------- Old: ---- openSUSE-release-tools-20171226.96a3eec.obscpio New: ---- openSUSE-release-tools-20180102.98a659c.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openSUSE-release-tools.spec ++++++ --- /var/tmp/diff_new_pack.qsXQKW/_old 2018-01-02 16:35:51.599022346 +0100 +++ /var/tmp/diff_new_pack.qsXQKW/_new 2018-01-02 16:35:51.603021012 +0100 @@ -1,7 +1,7 @@ # # spec file for package openSUSE-release-tools # -# Copyright (c) 2017 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2018 SUSE LINUX GmbH, Nuernberg, Germany. # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -20,7 +20,7 @@ %define source_dir osc-plugin-factory %define announcer_filename factory-package-news Name: openSUSE-release-tools -Version: 20171226.96a3eec +Version: 20180102.98a659c Release: 0 Summary: Tools to aid in staging and release work for openSUSE/SUSE License: GPL-2.0+ and MIT @@ -483,11 +483,6 @@ %files pkglistgen %defattr(-,root,root,-) %{_bindir}/osrt-pkglistgen -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0 -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0-all -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Rings -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Staging -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh %{_unitdir}/osrt-pkglistgen@.service %{_unitdir}/osrt-pkglistgen@.timer ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.qsXQKW/_old 2018-01-02 16:35:51.651005011 +0100 +++ /var/tmp/diff_new_pack.qsXQKW/_new 2018-01-02 16:35:51.651005011 +0100 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/openSUSE/osc-plugin-factory.git</param> - <param name="changesrevision">96a3eec425d49f995a85e000da4a8799b5a6a461</param> + <param name="changesrevision">98a659c3fab8dc036f9945854eae93265d695bed</param> </service> </servicedata> ++++++ openSUSE-release-tools-20171226.96a3eec.obscpio -> openSUSE-release-tools-20180102.98a659c.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/Makefile new/openSUSE-release-tools-20180102.98a659c/Makefile --- old/openSUSE-release-tools-20171226.96a3eec/Makefile 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/Makefile 2018-01-02 11:39:25.000000000 +0100 @@ -9,7 +9,7 @@ leaper.py \ manager_42.py \ metrics.py \ - pkglistgen.sh \ + pkglistgen.py \ repo_checker.py \ suppkg_rebuild.py \ totest-manager.py \ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/ReviewBot.py new/openSUSE-release-tools-20180102.98a659c/ReviewBot.py --- old/openSUSE-release-tools-20171226.96a3eec/ReviewBot.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/ReviewBot.py 2018-01-02 11:39:25.000000000 +0100 @@ -436,13 +436,15 @@ def comment_write(self, state='done', result=None, project=None, package=None, request=None, message=None, identical=False, only_replace=False, - info_extra=None): + info_extra=None, info_extra_identical=True): """Write comment if not similar to previous comment and replace old one. The state, result, and info_extra (dict) are combined to create the info that is passed to CommentAPI methods for creating a marker and finding previous comments. self.bot_name, which defaults to class, will be used - as the primary matching key. + as the primary matching key. When info_extra_identical is set to False + info_extra will not be included when finding previous comments to + compare message against. A comment from the same bot will be replaced when a new comment is written. The only_replace flag will restrict to only writing a comment @@ -482,15 +484,24 @@ message = '\n\n'.join(self.comment_handler.lines) info = {'state': state, 'result': result} - if info_extra: + if info_extra and info_extra_identical: info.update(info_extra) - message = self.comment_api.add_marker(message, self.bot_name, info) - message = self.comment_api.truncate(message.strip()) comments = self.comment_api.get_comments(**kwargs) comment, _ = self.comment_api.comment_find(comments, self.bot_name, info) + + if info_extra and not info_extra_identical: + # Add info_extra once comment has already been matched. + info.update(info_extra) + + message = self.comment_api.add_marker(message, self.bot_name, info) + message = self.comment_api.truncate(message.strip()) + if (comment is not None and - ((identical and comment['comment'] == message) or + ((identical and + # Remove marker from comments since handled during comment_find(). + self.comment_api.remove_marker(comment['comment']) == + self.comment_api.remove_marker(message)) or (not identical and comment['comment'].count('\n') == message.count('\n'))) ): # Assume same state/result and number of lines in message is duplicate. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/ToolBase.py new/openSUSE-release-tools-20180102.98a659c/ToolBase.py --- old/openSUSE-release-tools-20171226.96a3eec/ToolBase.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/ToolBase.py 2018-01-02 11:39:25.000000000 +0100 @@ -140,7 +140,7 @@ parser = cmdln.Cmdln.get_optparser(self) parser.add_option("--apiurl", '-A', metavar="URL", help="api url") parser.add_option("--dry", action="store_true", help="dry run") - parser.add_option("--debug", action="store_true", help="debug output") + parser.add_option("-d", "--debug", action="store_true", help="debug output") parser.add_option("--osc-debug", action="store_true", help="osc debug output") parser.add_option("--verbose", action="store_true", help="verbose") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/dist/package/openSUSE-release-tools.spec new/openSUSE-release-tools-20180102.98a659c/dist/package/openSUSE-release-tools.spec --- old/openSUSE-release-tools-20171226.96a3eec/dist/package/openSUSE-release-tools.spec 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/dist/package/openSUSE-release-tools.spec 2018-01-02 11:39:25.000000000 +0100 @@ -483,11 +483,6 @@ %files pkglistgen %defattr(-,root,root,-) %{_bindir}/osrt-pkglistgen -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0 -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0-all -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Rings -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Staging -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh %{_unitdir}/osrt-pkglistgen@.service %{_unitdir}/osrt-pkglistgen@.timer diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/fcc_submitter.py new/openSUSE-release-tools-20180102.98a659c/fcc_submitter.py --- old/openSUSE-release-tools-20171226.96a3eec/fcc_submitter.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/fcc_submitter.py 2018-01-02 11:39:25.000000000 +0100 @@ -184,10 +184,25 @@ root = ET.fromstring(''.join(f)) #print ET.dump(root) + failed_multibuild_pacs = [] pacs = [] for node in root.findall('result'): if node.get('repository') == 'pure_42' and node.get('arch') == 'x86_64': for pacnode in node.findall('status'): + if ':' in pacnode.get('package'): + mainpac = pacnode.get('package').split(':')[0] + if pacnode.get('code') not in ['succeeded', 'excluded']: + failed_multibuild_pacs.append(pacnode.get('package')) + if mainpac not in failed_multibuild_pacs: + failed_multibuild_pacs.append(mainpac) + if mainpac in pacs: + pacs.remove(mainpac) + else: + if mainpac in failed_multibuild_pacs: + failed_multibuild_pacs.append(pacnode.get('package')) + elif mainpac not in pacs: + pacs.append(mainpac) + continue if pacnode.get('code') == 'succeeded': pacs.append(pacnode.get('package')) else: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/manager_42.py new/openSUSE-release-tools-20180102.98a659c/manager_42.py --- old/openSUSE-release-tools-20171226.96a3eec/manager_42.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/manager_42.py 2018-01-02 11:39:25.000000000 +0100 @@ -46,7 +46,15 @@ class Manager42(object): config_defaults = { + 'ignored_packages' : [ + '00Meta', + '00aggregates', + '000product', + '000package-groups', + '000release-packages', + ], 'project_preference_order' : [], + 'drop_if_vanished_from' : [], 'from_prj' : 'openSUSE:Leap:42.3', 'factory' : 'openSUSE:Factory', } @@ -61,7 +69,7 @@ self.fill_package_meta() self.packages = dict() for project in [self.config.from_prj] + self.config.project_preference_order: - self.packages[project] = self.get_source_packages(project) + self._fill_package_list(project) # FIXME: add to ToolBase and rebase Manager42 on that def _load_config(self, handle = None): @@ -81,6 +89,9 @@ packages.add(title[3:].split(' ')[0]) return sorted(packages) + def all_packages(self): + return self.packages[self.config.from_prj] + def parse_lookup(self, project): self.lookup_changes = 0 self.lookup = {} @@ -152,11 +163,9 @@ ['source', project, package], opts)) - def crawl(self, given_packages = None): + def crawl(self, packages): """Main method of the class that runs the crawler.""" - packages = given_packages or self.packages[self.config.from_prj] - for package in sorted(packages): try: self.check_one_package(package) @@ -171,6 +180,18 @@ if self.lookup_changes: self.store_lookup() + def get_inconsistent(self): + known = set(self.lookup.keys()) + stale = known - set(self.packages[self.config.from_prj]) + unknown = set(self.packages[self.config.from_prj]) - known + + if (stale): + logger.info("stale packages: %s", ', '.join(stale)) + if (unknown): + logger.info("unknown packages: %s", ', '.join(unknown)) + + return (stale|unknown) + def get_package_history(self, project, package, deleted = False): try: query = {} @@ -183,9 +204,20 @@ return None raise - def check_source_in_project(self, project, package, verifymd5, deleted=False): + + def _is_ignored(self, project, package): + if package in self.config.ignored_packages: + logger.debug("%s in ignore list", package) + return True + return False + + def _fill_package_list(self, project): if project not in self.packages: - self.packages[project] = self.get_source_packages(project) + self.packages[project] = [ p for p in self.get_source_packages(project) if not self._is_ignored(project, p) ] + + def check_source_in_project(self, project, package, verifymd5, deleted=False): + + self._fill_package_list(project) if not deleted and not package in self.packages[project]: return None, None @@ -289,6 +321,8 @@ if not foundit: if lproject == 'FORK': logger.debug("{}: lookup is correctly marked as fork".format(package)) + elif lproject in self.config.drop_if_vanished_from: + logger.info('{} dropped from {}'.format(package, lproject)) else: logger.info('{} is a fork (was {})'.format(package, lproject)) self.lookup[package] = 'FORK' @@ -318,9 +352,15 @@ osc.conf.config['debug'] = args.debug uc = Manager42(caching = args.cache_requests, configfh = args.config ) - given_packages = args.packages - if not args.all and not given_packages: - given_packages = uc.latest_packages() + given_packages = set(args.packages) + if args.all: + given_packages = set(uc.all_packages()) + elif not given_packages: + given_packages = set(uc.latest_packages()) + + if args.check_inconsistent: + given_packages |= uc.get_inconsistent() + if args.force: uc.force = True uc.crawl(given_packages) @@ -342,6 +382,8 @@ help='don\'t take previous lookup information into consideration') parser.add_argument('--cache-requests', action='store_true', default=False, help='cache GET requests. Not recommended for daily use.') + parser.add_argument('--check-inconsistent', action='store_true', default=False, + help='also check insonsistent lookup entries') parser.add_argument("packages", nargs='*', help="packages to check") args = parser.parse_args() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/osclib/check_duplicate_binaries_command.py new/openSUSE-release-tools-20180102.98a659c/osclib/check_duplicate_binaries_command.py --- old/openSUSE-release-tools-20171226.96a3eec/osclib/check_duplicate_binaries_command.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/osclib/check_duplicate_binaries_command.py 2018-01-02 11:39:25.000000000 +0100 @@ -60,14 +60,19 @@ binaries[arch][name] = package + # convert sets to lists for readable yaml + for arch in duplicates.keys(): + for name in duplicates[arch].keys(): + duplicates[arch][name] = list(duplicates[arch][name]) + + current = yaml.dump(duplicates, default_flow_style=False) if save: args = ['{}:Staging'.format(self.api.project), 'dashboard', 'duplicate_binaries'] previous = self.api.load_file_content(*args) - current = yaml.dump(duplicates, default_flow_style=False) if current != previous: args.append(current) self.api.save_file_content(*args) else: - pprint(duplicates) + print(current) # vim: sw=4 et diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/osclib/comments.py new/openSUSE-release-tools-20180102.98a659c/osclib/comments.py --- old/openSUSE-release-tools-20171226.96a3eec/osclib/comments.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/osclib/comments.py 2018-01-02 11:39:25.000000000 +0100 @@ -123,6 +123,12 @@ marker = '<!-- {}{} -->'.format(bot, ' ' + ' '.join(infos) if info else '') return marker + '\n\n' + comment + def remove_marker(self, comment): + if comment.startswith(' Reply