Hello community, here is the log from the commit of package openSUSE-release-tools for openSUSE:Factory checked in at 2018-04-17 11:19:11 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old) and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "openSUSE-release-tools" Tue Apr 17 11:19:11 2018 rev:81 rq:597234 version:20180416.dc30723 Changes: -------- --- /work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes 2018-04-16 12:48:01.755207226 +0200 +++ /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new/openSUSE-release-tools.changes 2018-04-17 11:19:14.653554880 +0200 @@ -1,0 +2,13 @@ +Mon Apr 16 20:59:10 UTC 2018 - opensuse-releaseteam@opensuse.org + +- Update to version 20180416.dc30723: + * unmaintained: tool to find umaintained binaries sourced from SLE. + * osclib/core: provide source_file_load() from StagingAPI. + +------------------------------------------------------------------- +Fri Apr 13 20:15:51 UTC 2018 - opensuse-releaseteam@opensuse.org + +- Update to version 20180413.71fd5fe: + * pkglistgen: do_update_and_solve(): print exception and continue for all projects. + +------------------------------------------------------------------- Old: ---- openSUSE-release-tools-20180411.820d103.obscpio New: ---- openSUSE-release-tools-20180416.dc30723.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openSUSE-release-tools.spec ++++++ --- /var/tmp/diff_new_pack.MW7kD8/_old 2018-04-17 11:19:16.861451393 +0200 +++ /var/tmp/diff_new_pack.MW7kD8/_new 2018-04-17 11:19:16.865451205 +0200 @@ -20,7 +20,7 @@ %define source_dir openSUSE-release-tools %define announcer_filename factory-package-news Name: openSUSE-release-tools -Version: 20180411.820d103 +Version: 20180416.dc30723 Release: 0 Summary: Tools to aid in staging and release work for openSUSE/SUSE License: GPL-2.0-or-later AND MIT ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.MW7kD8/_old 2018-04-17 11:19:16.921448580 +0200 +++ /var/tmp/diff_new_pack.MW7kD8/_new 2018-04-17 11:19:16.921448580 +0200 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/openSUSE/openSUSE-release-tools.git</param> - <param name="changesrevision">1102e3cebcce638d3f2a241f6a9131482140619d</param> + <param name="changesrevision">dc30723b4a7f678dd9cd2195904ef472f49e93a1</param> </service> </servicedata> ++++++ openSUSE-release-tools-20180411.820d103.obscpio -> openSUSE-release-tools-20180416.dc30723.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180411.820d103/osclib/core.py new/openSUSE-release-tools-20180416.dc30723/osclib/core.py --- old/openSUSE-release-tools-20180411.820d103/osclib/core.py 2018-04-11 14:30:51.000000000 +0200 +++ new/openSUSE-release-tools-20180416.dc30723/osclib/core.py 2018-04-16 22:53:30.000000000 +0200 @@ -222,3 +222,13 @@ email = '{} <{}>'.format(realname.text, email) return email + +def source_file_load(apiurl, project, package, filename, revision=None): + query = {'expand': 1} + if revision: + query['rev'] = revision + url = makeurl(apiurl, ['source', project, package, filename], query) + try: + return http_GET(url).read() + except HTTPError: + return None diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180411.820d103/osclib/stagingapi.py new/openSUSE-release-tools-20180416.dc30723/osclib/stagingapi.py --- old/openSUSE-release-tools-20180411.820d103/osclib/stagingapi.py 2018-04-11 14:30:51.000000000 +0200 +++ new/openSUSE-release-tools-20180416.dc30723/osclib/stagingapi.py 2018-04-16 22:53:30.000000000 +0200 @@ -50,6 +50,7 @@ from osclib.cache import Cache from osclib.core import devel_project_get from osclib.core import project_list_prefix +from osclib.core import source_file_load from osclib.comments import CommentAPI from osclib.ignore_command import IgnoreCommand from osclib.memoize import memoize @@ -1506,14 +1507,7 @@ :param filename: The filename to query :param revision: The revision to query """ - query = {'expand': 1} - if revision: - query['rev'] = revision - url = self.makeurl(['source', project, package, filename], query) - try: - return http_GET(url).read() - except urllib2.HTTPError: - return None + return source_file_load(self.apiurl, project, package, filename, revision) def save_file_content(self, project, package, filename, content, comment='script updated'): """ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180411.820d103/pkglistgen.py new/openSUSE-release-tools-20180416.dc30723/pkglistgen.py --- old/openSUSE-release-tools-20180411.820d103/pkglistgen.py 2018-04-11 14:30:51.000000000 +0200 +++ new/openSUSE-release-tools-20180416.dc30723/pkglistgen.py 2018-04-16 22:53:30.000000000 +0200 @@ -58,6 +58,7 @@ import urlparse import gzip import tempfile +import traceback import random import shutil import string @@ -1110,6 +1111,8 @@ ${cmd_option_list} """ + self.error_occured = False + if not opts.project: raise ValueError('project is required') if opts.scope not in self.SCOPES: @@ -1119,7 +1122,7 @@ for scope in self.SCOPES[1:]: opts.scope = scope self.do_update_and_solve(subcmd, copy.deepcopy(opts)) - return + return self.error_occured # Store target project as opts.project will contain subprojects. target_project = opts.project @@ -1137,27 +1140,27 @@ if opts.scope == 'target': self.options.repos = ['/'.join([target_project, main_repo])] - self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts, drop_list=True) - return + self.update_and_solve_target_wrapper(apiurl, target_project, target_config, main_repo, opts, drop_list=True) + return self.error_occured elif opts.scope == 'ports': # TODO Continue supporting #1297, but should be abstracted. main_repo = 'ports' opts.project += ':Ports' self.options.repos = ['/'.join([opts.project, main_repo])] - self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts, drop_list=True) - return + self.update_and_solve_target_wrapper(apiurl, target_project, target_config, main_repo, opts, drop_list=True) + return self.error_occured elif opts.scope == 'rings': opts.project = api.rings[1] self.options.repos = [ '/'.join([api.rings[1], main_repo]), '/'.join([api.rings[0], main_repo]), ] - self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts) + self.update_and_solve_target_wrapper(apiurl, target_project, target_config, main_repo, opts) opts.project = api.rings[2] self.options.repos.insert(0, '/'.join([api.rings[2], main_repo])) - self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts, skip_release=True) - return + self.update_and_solve_target_wrapper(apiurl, target_project, target_config, main_repo, opts, skip_release=True) + return self.error_occured elif opts.scope == 'staging': letters = api.get_staging_projects_short() for letter in letters: @@ -1172,11 +1175,23 @@ opts_dvd = copy.deepcopy(opts) opts_dvd.project += ':DVD' self.options.repos.insert(0, '/'.join([opts_dvd.project, main_repo])) - self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts_dvd, skip_release=True) + self.update_and_solve_target_wrapper(apiurl, target_project, target_config, main_repo, opts_dvd, skip_release=True) self.options.repos.pop(0) - self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts) - return + self.update_and_solve_target_wrapper(apiurl, target_project, target_config, main_repo, opts) + return self.error_occured + + def update_and_solve_target_wrapper(self, *args, **kwargs): + try: + self.update_and_solve_target(*args, **kwargs) + except Exception as e: + # Print exception, but continue to prevent problems effecting one + # project from killing the whole process. Downside being a common + # error will be duplicated for each project. Common exceptions could + # be excluded if a set list is determined, but that is likely not + # practical. + traceback.print_exc() + self.error_occured = True def update_and_solve_target(self, apiurl, target_project, target_config, main_repo, opts, skip_release=False, drop_list=False): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180411.820d103/unmaintained.py new/openSUSE-release-tools-20180416.dc30723/unmaintained.py --- old/openSUSE-release-tools-20180411.820d103/unmaintained.py 1970-01-01 01:00:00.000000000 +0100 +++ new/openSUSE-release-tools-20180416.dc30723/unmaintained.py 2018-04-16 22:53:30.000000000 +0200 @@ -0,0 +1,81 @@ +#!/usr/bin/python + +import argparse +from lxml import etree as ET +from osc import conf +from osc.core import meta_get_filelist +from osclib.core import package_binary_list +from osclib.core import source_file_load +import sys +import yaml + +def kiwi_binaries(apiurl, project): + binaries = set() + for filename in meta_get_filelist(apiurl, project, '000product'): + if not filename.endswith('.kiwi'): + continue + + kiwi = ET.fromstring(source_file_load( + apiurl, project, '000product', filename)) + + binaries.update(kiwi.xpath('//instsource/repopackages/repopackage/@name')) + + return binaries + +def unmaintained(apiurl, project_target): + lookup = yaml.safe_load(source_file_load( + apiurl, project_target, '00Meta', 'lookup.yml')) + lookup_total = len(lookup) + lookup = {k: v for k, v in lookup.iteritems() if v.startswith('SUSE:SLE')} + + package_binaries, _ = package_binary_list( + apiurl, project_target, 'standard', 'x86_64') + package_binaries_total = len(package_binaries) + package_binaries = [pb for pb in package_binaries if pb.arch != 'src' and pb.package in lookup] + + # Determine max length possible for each column. + maxes = [ + len(max([b.name for b in package_binaries], key=len)), + len(max(lookup.keys(), key=len)), + len(max(lookup.values(), key=len)), + ] + line_format = ' '.join(['{:<' + str(m) + '}' for m in maxes]) + + print(line_format.format('binary', 'package', 'source project')) + + project_sources = {} + binaries_unmaintained = 0 + packages_unmaintained = set() + for package_binary in sorted(package_binaries, key=lambda pb: pb.name): + project_source = lookup[package_binary.package] + if project_source not in project_sources: + # Load binaries referenced in kiwi the first time source encountered. + project_sources[project_source] = kiwi_binaries(apiurl, project_source) + + if package_binary.name not in project_sources[project_source]: + print(line_format.format( + package_binary.name, package_binary.package, project_source)) + + binaries_unmaintained += 1 + packages_unmaintained.add(package_binary.package) + + print('{:,} of {:,} binaries ({:,} packages) unmaintained from SLE of {:,} total binaries ({:,} packages) in project'.format( + binaries_unmaintained, len(package_binaries), len(packages_unmaintained), package_binaries_total, lookup_total)) + +def main(args): + conf.get_config(override_apiurl=args.apiurl) + conf.config['debug'] = args.debug + apiurl = conf.config['apiurl'] + + return not unmaintained(apiurl, args.project_target) + + +if __name__ == '__main__': + description = 'Review each binary in target project sourced from SLE to see if utilized in kiwi files.' + parser = argparse.ArgumentParser(description=description) + parser.add_argument('-A', '--apiurl', help='OBS instance API URL') + parser.add_argument('-d', '--debug', action='store_true', help='print useful debugging info') + parser.add_argument('project_target', help='target project to search') + args = parser.parse_args() + + sys.exit(main(args)) ++++++ openSUSE-release-tools.obsinfo ++++++ --- /var/tmp/diff_new_pack.MW7kD8/_old 2018-04-17 11:19:17.569418209 +0200 +++ /var/tmp/diff_new_pack.MW7kD8/_new 2018-04-17 11:19:17.573418022 +0200 @@ -1,5 +1,5 @@ name: openSUSE-release-tools -version: 20180411.820d103 -mtime: 1523449851 -commit: 820d1030e54f5c9bbfe9aeb69ca5b3b44a838aaa +version: 20180416.dc30723 +mtime: 1523912010 +commit: dc30723b4a7f678dd9cd2195904ef472f49e93a1