Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package openSUSE-release-tools for openSUSE:Factory checked in at 2021-08-31 19:55:38 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old) and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.1899 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "openSUSE-release-tools" Tue Aug 31 19:55:38 2021 rev:318 rq:915210 version:20210831.0de63caa Changes: -------- --- /work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes 2021-08-23 10:09:10.456176120 +0200 +++ /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.1899/openSUSE-release-tools.changes 2021-08-31 19:56:32.054025267 +0200 @@ -1,0 +2,8 @@ +Tue Aug 31 08:53:00 UTC 2021 - opensuse-releaseteam@opensuse.org + +- Update to version 20210831.0de63caa: + * Update specfile for skippkg-finder + * Run skippkg-finder.py before pkglistgen + * Add skippkg-finder.py to overwrite NON_FTP_PACKAGES for obsoleted and unneeded package + +------------------------------------------------------------------- Old: ---- openSUSE-release-tools-20210820.43b07171.obscpio New: ---- openSUSE-release-tools-20210831.0de63caa.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openSUSE-release-tools.spec ++++++ --- /var/tmp/diff_new_pack.NbnALe/_old 2021-08-31 19:56:32.726026196 +0200 +++ /var/tmp/diff_new_pack.NbnALe/_new 2021-08-31 19:56:32.730026202 +0200 @@ -20,7 +20,7 @@ %define source_dir openSUSE-release-tools %define announcer_filename factory-package-news Name: openSUSE-release-tools -Version: 20210820.43b07171 +Version: 20210831.0de63caa Release: 0 Summary: Tools to aid in staging and release work for openSUSE/SUSE License: GPL-2.0-or-later AND MIT @@ -438,6 +438,7 @@ %exclude %{_datadir}/%{source_dir}/maintenance-installcheck.py %exclude %{_datadir}/%{source_dir}/project-installcheck.py %exclude %{_datadir}/%{source_dir}/suppkg_rebuild.py +%exclude %{_datadir}/%{source_dir}/skippkg-finder.py %exclude %{_datadir}/%{source_dir}/osclib %exclude %{_datadir}/%{source_dir}/osc-cycle.py %exclude %{_datadir}/%{source_dir}/osc-origin.py @@ -551,8 +552,10 @@ %files pkglistgen %defattr(-,root,root,-) %{_bindir}/osrt-pkglistgen +%{_bindir}/osrt-skippkg-finder %{_datadir}/%{source_dir}/pkglistgen %{_datadir}/%{source_dir}/pkglistgen.py +%{_datadir}/%{source_dir}/skippkg-finder.py %files -n osclib %defattr(-,root,root,-) ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.NbnALe/_old 2021-08-31 19:56:32.782026274 +0200 +++ /var/tmp/diff_new_pack.NbnALe/_new 2021-08-31 19:56:32.786026280 +0200 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/openSUSE/openSUSE-release-tools.git</param> - <param name="changesrevision">33d44e218e9251d5dd3cc30d285c03a5a20b5b4c</param> + <param name="changesrevision">2a3ee0c246daccbd3b500896fe6057c8dc04777b</param> </service> </servicedata> ++++++ openSUSE-release-tools-20210820.43b07171.obscpio -> openSUSE-release-tools-20210831.0de63caa.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20210820.43b07171/dist/package/openSUSE-release-tools.spec new/openSUSE-release-tools-20210831.0de63caa/dist/package/openSUSE-release-tools.spec --- old/openSUSE-release-tools-20210820.43b07171/dist/package/openSUSE-release-tools.spec 2021-08-20 09:49:06.000000000 +0200 +++ new/openSUSE-release-tools-20210831.0de63caa/dist/package/openSUSE-release-tools.spec 2021-08-31 10:51:24.000000000 +0200 @@ -439,6 +439,7 @@ %exclude %{_datadir}/%{source_dir}/maintenance-installcheck.py %exclude %{_datadir}/%{source_dir}/project-installcheck.py %exclude %{_datadir}/%{source_dir}/suppkg_rebuild.py +%exclude %{_datadir}/%{source_dir}/skippkg-finder.py %exclude %{_datadir}/%{source_dir}/osclib %exclude %{_datadir}/%{source_dir}/osc-cycle.py %exclude %{_datadir}/%{source_dir}/osc-origin.py @@ -552,8 +553,10 @@ %files pkglistgen %defattr(-,root,root,-) %{_bindir}/osrt-pkglistgen +%{_bindir}/osrt-skippkg-finder %{_datadir}/%{source_dir}/pkglistgen %{_datadir}/%{source_dir}/pkglistgen.py +%{_datadir}/%{source_dir}/skippkg-finder.py %files -n osclib %defattr(-,root,root,-) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20210820.43b07171/gocd/pkglistgen.opensuse.gocd.yaml new/openSUSE-release-tools-20210831.0de63caa/gocd/pkglistgen.opensuse.gocd.yaml --- old/openSUSE-release-tools-20210820.43b07171/gocd/pkglistgen.opensuse.gocd.yaml 2021-08-20 09:49:06.000000000 +0200 +++ new/openSUSE-release-tools-20210831.0de63caa/gocd/pkglistgen.opensuse.gocd.yaml 2021-08-31 10:51:24.000000000 +0200 @@ -107,6 +107,7 @@ resources: - repo-checker tasks: + - script: python3 ./skippkg-finder.py-A https://api.opensuse.org -o openSUSE:Leap:15.4 -s SUSE:SLE-15-SP4:GA - script: python3 ./pkglistgen.py -d -A https://api.opensuse.org update_and_solve -p openSUSE:Leap:15.4 -s target Update.Repos.Leap: group: Leap.pkglistgen diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20210820.43b07171/gocd/pkglistgen.opensuse.gocd.yaml.erb new/openSUSE-release-tools-20210831.0de63caa/gocd/pkglistgen.opensuse.gocd.yaml.erb --- old/openSUSE-release-tools-20210820.43b07171/gocd/pkglistgen.opensuse.gocd.yaml.erb 2021-08-20 09:49:06.000000000 +0200 +++ new/openSUSE-release-tools-20210831.0de63caa/gocd/pkglistgen.opensuse.gocd.yaml.erb 2021-08-31 10:51:24.000000000 +0200 @@ -81,6 +81,7 @@ resources: - repo-checker tasks: + - script: python3 ./skippkg-finder.py-A https://api.opensuse.org -o openSUSE:Leap:15.4 -s SUSE:SLE-15-SP4:GA - script: python3 ./pkglistgen.py -d -A https://api.opensuse.org update_and_solve -p <%= project[0] %><%= options %> <% end -%> Update.Repos.Leap: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20210820.43b07171/skippkg-finder.py new/openSUSE-release-tools-20210831.0de63caa/skippkg-finder.py --- old/openSUSE-release-tools-20210820.43b07171/skippkg-finder.py 1970-01-01 01:00:00.000000000 +0100 +++ new/openSUSE-release-tools-20210831.0de63caa/skippkg-finder.py 2021-08-31 10:51:24.000000000 +0200 @@ -0,0 +1,372 @@ +#!/usr/bin/python3 + +import argparse +import logging +import sys + +from urllib.error import HTTPError + +import re +from lxml import etree as ET +from collections import namedtuple + +import osc.conf +import osc.core +from osc.core import http_GET +from osc.core import makeurl +from osc import oscerr +import osclib +from osclib.core import source_file_ensure + +SUPPORTED_ARCHS = ['x86_64', 'i586', 'aarch64', 'ppc64le', 's390x'] +DEFAULT_REPOSITORY = 'standard' + +META_PACKAGE = '000package-groups' + +class SkippkgFinder(object): + def __init__(self, opensuse_project, sle_project, print_only, verbose): + self.opensuse_project = opensuse_project + self.sle_project = sle_project + self.print_only = print_only + self.verbose = verbose + self.apiurl = osc.conf.config['apiurl'] + self.debug = osc.conf.config['debug'] + + def is_sle_specific(self, package): + """ + Return True if package is provided for SLE only or a SLE forking. + Add new condition here if you do not want package being added to + selected_binarylist[]. + """ + pkg = package.lower() + prefixes = ( + 'desktop-data', + 'libyui-bindings', + 'libyui-doc', + 'libyui-ncurses', + 'libyui-qt', + 'libyui-rest', + 'lifecycle-data-sle', + 'kernel-livepatch', + 'kiwi-template', + 'mgr-', + 'migrate', + 'patterns', + 'release-notes', + 'sap', + 'sca-', + 'skelcd', + 'sle-', + 'sle_', + 'sle15', + 'sles15', + 'spacewalk', + 'supportutils-plugin', + 'suse-migration', + 'susemanager-', + 'yast2-hana' + ) + suffixes = ( + '-caasp', + '-sle', + 'bootstrap' + ) + matches = ( + 'gtk-vnc2', + 'ibus-googlepinyin', + 'infiniband-diags', + 'llvm', + 'lua51-luajit', + 'lvm2-clvm', + 'osad', + 'rhncfg', + 'python-ibus', + 'python-pymemcache', + 'suse-build-key', + 'suse-hpc', + 'txt2tags', + 'zypp-plugin-spacewalk', + 'zypper-search-packages-plugin' + ) + if pkg.startswith(prefixes) or pkg.endswith(suffixes) or pkg in matches: + return True + if 'sles' in pkg or\ + 'sled' in pkg or\ + 'sap-' in pkg or\ + '-sap' in pkg or\ + 'eula' in pkg or\ + 'branding' in pkg: + return True + return False + + def get_packagelist(self, project, by_project=True): + """ + Return the list of package's info of a project. + If the latest package is from an incident then returns incident + package. + """ + + pkglist = {} + packageinfo = {} + query = {'expand': 1} + root = ET.parse(http_GET(makeurl(self.apiurl, ['source', project], + query=query))).getroot() + for i in root.findall('entry'): + pkgname = i.get('name') + orig_project = i.get('originproject') + is_incidentpkg = False + # Metapackage should not be selected + if pkgname.startswith('000') or\ + pkgname.startswith('_') or\ + pkgname.startswith('patchinfo.') or\ + pkgname.startswith('skelcd-') or\ + pkgname.startswith('installation-images') or\ + pkgname.endswith('-mini'): + continue + # Ugly hack for package has dot in source package name + # eg. go1.x incidents as the name would be go1.x.xxx + if '.' in pkgname and re.match(r'[0-9]+$', pkgname.split('.')[-1]) and \ + orig_project.startswith('SUSE:') and orig_project.endswith(':Update'): + is_incidentpkg = True + if pkgname.startswith('go1') or\ + pkgname.startswith('bazel0') or\ + pkgname.startswith('dotnet') or\ + pkgname.startswith('rust1') or\ + pkgname.startswith('ruby2'): + if not (pkgname.count('.') > 1): + is_incidentpkg = False + + # If an incident found then update the package origin info + if is_incidentpkg: + orig_name = re.sub(r'\.[0-9]+$', '', pkgname) + incident_number = int(pkgname.split('.')[-1]) + if orig_name in pkglist and pkglist[orig_name]['Project'] == orig_project: + if re.match(r'[0-9]+$', pkglist[orig_name]['Package'].split('.')[-1]): + old_incident_number = int(pkglist[orig_name]['Package'].split('.')[-1]) + if incident_number > old_incident_number: + pkglist[orig_name]['Package'] = pkgname + else: + pkglist[orig_name]['Package'] = pkgname + else: + pkglist[pkgname] = {'Project': orig_project, 'Package': pkgname} + + if by_project: + for pkg in pkglist.keys(): + if pkglist[pkg]['Project'].startswith('SUSE:') and self.is_sle_specific(pkg): + continue + if pkglist[pkg]['Project'] not in packageinfo: + packageinfo[pkglist[pkg]['Project']] = [] + if pkglist[pkg]['Package'] not in packageinfo[pkglist[pkg]['Project']]: + packageinfo[pkglist[pkg]['Project']].append(pkglist[pkg]['Package']) + return packageinfo + + return pkglist + + def get_project_binary_list(self, project, repository, arch, package_binaries={}): + """ + Returns binarylist of a project + """ + + # Use pool repository for SUSE namespace project. + # Because RPMs were injected to pool repository on OBS rather than + # standard repository. + if project.startswith('SUSE:'): + repository = 'pool' + + path = ['build', project, repository, arch] + url = makeurl(self.apiurl, path, {'view': 'binaryversions'}) + root = ET.parse(http_GET(url)).getroot() + + for binary_list in root: + package = binary_list.get('package') + package = package.split(':', 1)[0] + index = project + "_" + package + + if index not in package_binaries: + package_binaries[index] = [] + for binary in binary_list: + filename = binary.get('name') + result = re.match(osclib.core.RPM_REGEX, filename) + if not result: + continue + + if result.group('arch') == 'src' or result.group('arch') == 'nosrc': + continue + if result.group('name').endswith('-debuginfo') or result.group('name').endswith('-debuginfo-32bit'): + continue + if result.group('name').endswith('-debugsource'): + continue + + if result.group('name') not in package_binaries[index]: + package_binaries[index].append(result.group('name')) + + return package_binaries + + def exception_package(self, package): + """ + Do not skip the package if matches the condition. + package parameter is source package name. + """ + + if '-bootstrap' in package or\ + 'Tumbleweed' in package or\ + 'metis' in package: + return True + # These packages must have a good reason not to be single-speced + # from one source. + if package.startswith('python2-') or\ + package.startswith('python3'): + return True + return False + + def exception_binary(self, package): + """ + Do not skip the binary if matches the condition + package parameter is RPM filename. + """ + + if package == 'openSUSE-release' or\ + package == 'openSUSE-release-ftp' or\ + package == 'openSUSE-Addon-NonOss-release': + return True + return False + + def crawl(self): + """Main method""" + + leap_pkglist = self.get_packagelist(self.opensuse_project) + sle_pkglist = self.get_packagelist(self.sle_project, by_project=False) + # The selected_binarylist[] includes the latest sourcepackage list + # binary RPMs from the latest sources need to be presented in ftp eventually + selected_binarylist = [] + # Any existed binary RPMs from any SPx/Leap/Backports + fullbinarylist = [] + # package_binaries[] is a pre-formated binarylist per each package + # access to the conotent uses package_binaries['SUSE:SLE-15:Update_libcdio.12032'] + package_binaries = {} + + # Inject binarylist to a list per package name no matter what archtectures was + for arch in SUPPORTED_ARCHS: + for prj in leap_pkglist.keys(): + package_binaries = self.get_project_binary_list(prj, DEFAULT_REPOSITORY, arch, package_binaries) + + for pkg in package_binaries.keys(): + if not self.exception_package(pkg): + fullbinarylist += package_binaries[pkg] + + for prj in leap_pkglist.keys(): + for pkg in leap_pkglist[prj]: + cands = [prj + "_" + pkg] + # Handling for SLE forks, or package has different multibuild bits + # enablility between SLE and openSUSE + if prj.startswith('openSUSE:') and pkg in sle_pkglist and\ + not self.is_sle_specific(pkg): + cands.append(sle_pkglist[pkg]['Project'] + "_" + sle_pkglist[pkg]['Package']) + logging.debug(cands) + for index in cands: + if index in package_binaries: + selected_binarylist += package_binaries[index] + else: + logging.info("Can not find binary of %s" % index) + + # Some packages has been obsoleted by new updated package, however + # there are application still depend on old library when it builds + # eg. SUSE:SLE-15-SP3:GA has qpdf/libqpdf28 but cups-filter was build + # in/when SLE15 SP2 which requiring qpdf/libqpdf6, therefore old + # qpdf/libqpdf6 from SLE15 SP2 should not to be missed. + extra_packagelist = [ + # gnome-software requirement + 'SUSE:SLE-15-SP2:Update_libxmlb.15999', + # cups-filter requirement + 'SUSE:SLE-15-SP2:GA_qpdf', + # libcdio_paranoia2 requirement + 'SUSE:SLE-15:Update_libcdio.12032', + # libstoken1 requirement + 'SUSE:SLE-15:Update_libnettle.19992', + # python2-Pillow requirement + 'SUSE:SLE-15:Update_libwebp.19719', + # amarok requirement + 'SUSE:SLE-15:Update_mariadb.20531', + # bogofilter requirement + 'SUSE:SLE-15:GA_gsl', + # gnome-builder requirement + 'SUSE:SLE-15-SP2:GA_vala', + # hfst-ospell requirement + 'SUSE:SLE-15:Update_icu.14528' + ] + for pkg in extra_packagelist: + selected_binarylist += package_binaries[pkg] + + # Preparing a packagelist for the skipping candidate + obsoleted = [] + for pkg in fullbinarylist: + if pkg not in selected_binarylist and pkg not in obsoleted: + if not self.exception_binary(pkg): + obsoleted.append(pkg) + + # Post processing of obsoleted packagelist + tmp_obsoleted = obsoleted.copy() + for pkg in tmp_obsoleted: + # Respect to single-speced python package, when a python2 RPM is + # considered then a python3 flavor should also be selected to be + # skipped, if not, don't add it. + if pkg.startswith('python2-') and re.sub(r'^python2', 'python3', pkg) not in obsoleted: + obsoleted.remove(pkg) + # Main RPM must to be skipped if -32 bit RPM or -64bit RPM is + # considered. + if pkg.endswith('-32bit') or pkg.endswith('-64bit'): + main_filename = re.sub('-[36][24]bit', '', pkg) + if main_filename not in obsoleted: + obsoleted.remove(pkg) + + skip_list = ET.Element('group', {'name': 'NON_FTP_PACKAGES'}) + ET.SubElement(skip_list, 'conditional', {'name': 'drop_from_ftp'}) + packagelist = ET.SubElement(skip_list, 'packagelist', {'relationship': 'requires'}) + for pkg in sorted(obsoleted): + if not self.print_only and self.verbose: + print(pkg) + attr = {'name': pkg} + ET.SubElement(packagelist, 'package', attr) + if not self.print_only: + source_file_ensure(self.apiurl, self.opensuse_project, META_PACKAGE, 'NON_FTP_PACKAGES.group', + ET.tostring(skip_list, pretty_print=True, encoding='unicode'), + 'Update the skip list') + else: + print(ET.tostring(skip_list, pretty_print=True, + encoding='unicode')) + + +def main(args): + osc.conf.get_config(override_apiurl=args.apiurl) + osc.conf.config['debug'] = args.debug + + if args.opensuse_project is None or args.sle_project is None: + print("Please pass --opensuse-project and --sle-project argument. See usage with --help.") + quit() + + uc = SkippkgFinder(args.opensuse_project, args.sle_project, args.print_only, args.verbose) + uc.crawl() + + +if __name__ == '__main__': + description = 'Overwrites NON_FTP_PACKAGES.group according to the latest sources. '\ + 'This tool only works for Leap after CtLG implemented.' + parser = argparse.ArgumentParser(description=description) + parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL') + parser.add_argument('-d', '--debug', action='store_true', + help='print info useful for debuging') + parser.add_argument('-o', '--opensuse-project', dest='opensuse_project', metavar='OPENSUSE_PROJECT', + help='openSUSE project on buildservice') + parser.add_argument('-s', '--sle-project', dest='sle_project', metavar='SLE_PROJECT', + help='SLE project on buildservice') + parser.add_argument('-p', '--print-only', action='store_true', + help='show the result instead of the uploading') + parser.add_argument('-v', '--verbose', action='store_true', + help='show the diff') + + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG if args.debug + else logging.INFO) + + sys.exit(main(args)) ++++++ openSUSE-release-tools.obsinfo ++++++ --- /var/tmp/diff_new_pack.NbnALe/_old 2021-08-31 19:56:33.630027448 +0200 +++ /var/tmp/diff_new_pack.NbnALe/_new 2021-08-31 19:56:33.630027448 +0200 @@ -1,5 +1,5 @@ name: openSUSE-release-tools -version: 20210820.43b07171 -mtime: 1629445746 -commit: 43b07171ad85c0edbf2f11ec0853a32672e378d3 +version: 20210831.0de63caa +mtime: 1630399884 +commit: 0de63caa85db778ad9f6909093f87bedb11cad02