Hello community, here is the log from the commit of package openSUSE-release-tools for openSUSE:Factory checked in at 2019-07-29 17:31:24 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old) and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.4126 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "openSUSE-release-tools" Mon Jul 29 17:31:24 2019 rev:196 rq:719513 version:20190729.70133114 Changes: -------- --- /work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes 2019-07-26 17:34:22.336086142 +0200 +++ /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.4126/openSUSE-release-tools.changes 2019-07-29 17:31:27.350167588 +0200 @@ -1,0 +2,8 @@ +Mon Jul 29 08:24:07 UTC 2019 - opensuse-releaseteam@opensuse.org + +- Update to version 20190729.70133114: + * Kill rebuildpacs.pl in gocd config + * Rely on remote config and storage + * Replace rebuildpacs.pl with project-installcheck functionality + +------------------------------------------------------------------- Old: ---- openSUSE-release-tools-20190726.df07bcc2.obscpio New: ---- openSUSE-release-tools-20190729.70133114.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openSUSE-release-tools.spec ++++++ --- /var/tmp/diff_new_pack.2amClQ/_old 2019-07-29 17:31:28.078166707 +0200 +++ /var/tmp/diff_new_pack.2amClQ/_new 2019-07-29 17:31:28.078166707 +0200 @@ -20,7 +20,7 @@ %define source_dir openSUSE-release-tools %define announcer_filename factory-package-news Name: openSUSE-release-tools -Version: 20190726.df07bcc2 +Version: 20190729.70133114 Release: 0 Summary: Tools to aid in staging and release work for openSUSE/SUSE License: GPL-2.0-or-later AND MIT @@ -426,7 +426,6 @@ %{_bindir}/osrt-legal-auto %{_bindir}/osrt-obs_clone %{_bindir}/osrt-openqa-maintenance -%{_bindir}/osrt-rebuildpacs %{_bindir}/osrt-requestfinder %{_bindir}/osrt-scan_baselibs %{_bindir}/osrt-status ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.2amClQ/_old 2019-07-29 17:31:28.122166655 +0200 +++ /var/tmp/diff_new_pack.2amClQ/_new 2019-07-29 17:31:28.122166655 +0200 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/openSUSE/openSUSE-release-tools.git</param> - <param name="changesrevision">700bb203a52ccba5228b77cfac6d0b4a8249efc2</param> + <param name="changesrevision">071bbf8d66c4120446ca65ed41ba9ae4d811adc1</param> </service> </servicedata> ++++++ openSUSE-release-tools-20190726.df07bcc2.obscpio -> openSUSE-release-tools-20190729.70133114.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20190726.df07bcc2/CreatePackageDescr.pm new/openSUSE-release-tools-20190729.70133114/CreatePackageDescr.pm --- old/openSUSE-release-tools-20190726.df07bcc2/CreatePackageDescr.pm 2019-07-26 13:15:55.000000000 +0200 +++ new/openSUSE-release-tools-20190729.70133114/CreatePackageDescr.pm 2019-07-29 10:19:25.000000000 +0200 @@ -9,16 +9,18 @@ use Rpm; use Fcntl qw/:flock/; -sub package_snippet($) { +sub package_snippet { - my $package = shift; + my ($package) = @_; + # mark as used + utime undef, undef, $package; my $cachedir = dirname($package) . "/.cache/"; - my $cachefile = $cachedir . basename($package); + my $cachefile = $cachedir . '2-' . basename($package); my $out = ''; - if ( -f $cachefile ) { - open( C, '<', $cachefile ) || die "no cache for $package"; + if (-f $cachefile) { + open(C, '<', $cachefile) || die "no cache for $package"; flock(C, LOCK_SH) or die "failed to lock $cachefile: $!\n"; while (<C>) { $out .= $_; @@ -42,50 +44,66 @@ my %qq = Build::Rpm::rpmq( $package, - qw{NAME VERSION RELEASE ARCH OLDFILENAMES DIRNAMES BASENAMES DIRINDEXES 1030 1037 1039 1040 + qw{NAME VERSION RELEASE ARCH OLDFILENAMES DIRNAMES BASENAMES DIRINDEXES SOURCERPM 1030 1037 1039 1040 1047 1112 1113 1049 1048 1050 1090 1114 1115 1054 1053 1055 1036 5046 5047 5048 5049 5050 5051 - 5052 5053 5054 5055 5056 5057 1156 1158 1157 1159 1161 1160 + 5052 5053 5054 5055 5056 5057 1156 1158 1157 1159 1161 1160 1123 } ); if (!exists $qq{'NAME'}[0]) { print STDERR "corrupt rpm: $package\n"; unlink($package); - return $out; # Needs to be re-mirrored, so return blank to trigger error. + return $out; # Needs to be re-mirrored, so return blank to trigger error. } my $name = $qq{'NAME'}[0]; - Build::Rpm::add_flagsvers( \%qq, 1049, 1048, 1050 ); # requires - Build::Rpm::add_flagsvers( \%qq, 1047, 1112, 1113 ); # provides - Build::Rpm::add_flagsvers( \%qq, 1090, 1114, 1115 ); # obsoletes - Build::Rpm::add_flagsvers( \%qq, 1054, 1053, 1055 ); # conflicts - - Build::Rpm::add_flagsvers(\%qq, 1156, 1158, 1157) if $qq{1156}; # oldsuggests - Build::Rpm::add_flagsvers(\%qq, 1159, 1161, 1160) if $qq{1159}; # oldenhances - - Build::Rpm::add_flagsvers(\%qq, 5046, 5048, 5047) if $qq{5046}; # recommends - Build::Rpm::add_flagsvers(\%qq, 5049, 5051, 5050) if $qq{5049}; # suggests - Build::Rpm::add_flagsvers(\%qq, 5052, 5054, 5053) if $qq{5052}; # supplements - Build::Rpm::add_flagsvers(\%qq, 5055, 5057, 5056) if $qq{5055}; # enhances + Build::Rpm::add_flagsvers(\%qq, 1049, 1048, 1050); # requires + Build::Rpm::add_flagsvers(\%qq, 1047, 1112, 1113); # provides + Build::Rpm::add_flagsvers(\%qq, 1090, 1114, 1115); # obsoletes + Build::Rpm::add_flagsvers(\%qq, 1054, 1053, 1055); # conflicts + + Build::Rpm::add_flagsvers(\%qq, 1156, 1158, 1157) if $qq{1156}; # oldsuggests + Build::Rpm::add_flagsvers(\%qq, 1159, 1161, 1160) if $qq{1159}; # oldenhances + + Build::Rpm::add_flagsvers(\%qq, 5046, 5048, 5047) if $qq{5046}; # recommends + Build::Rpm::add_flagsvers(\%qq, 5049, 5051, 5050) if $qq{5049}; # suggests + Build::Rpm::add_flagsvers(\%qq, 5052, 5054, 5053) if $qq{5052}; # supplements + Build::Rpm::add_flagsvers(\%qq, 5055, 5057, 5056) if $qq{5055}; # enhances $arch = $qq{'ARCH'}[0]; # some packages are more equal than others $arch = 'i586' if $arch eq 'i686'; - $out .= sprintf( "=Pkg: %s %s %s %s\n", - $name, $qq{'VERSION'}[0], $qq{'RELEASE'}[0], $arch ); + $out .= sprintf("=Pkg: %s %s %s %s\n", + $name, $qq{'VERSION'}[0], $qq{'RELEASE'}[0], $arch); $out .= "+Flx:\n"; - my @modes = @{ $qq{1030} || [] }; - my @basenames = @{ $qq{BASENAMES} || [] }; - my @dirs = @{ $qq{DIRNAMES} || [] }; - my @dirindexes = @{ $qq{DIRINDEXES} || [] }; - my @users = @{ $qq{1039} || [] }; - my @groups = @{ $qq{1040} || [] }; - my @flags = @{ $qq{1037} || [] }; - my @linktos = @{ $qq{1036} || [] }; + my @modes = @{$qq{1030} || []}; + my @basenames = @{$qq{BASENAMES} || []}; + my @dirs = @{$qq{DIRNAMES} || []}; + my @dirindexes = @{$qq{DIRINDEXES} || []}; + my @users = @{$qq{1039} || []}; + my @groups = @{$qq{1040} || []}; + my @flags = @{$qq{1037} || []}; + my @linktos = @{$qq{1036} || []}; my @xprvs; + # e.g. libqt5-qttools-5.12.3-1.2.src.rpm + my $sourcerpm = $qq{SOURCERPM}->[0]; + my @source; + if ($sourcerpm =~ m/^(.*)-([^-]*)-([^-]*)\.(src|nosrc)\.rpm/) { + @source = ($1, $2, $3); + } + + # overwrite the source with the disturl, in case of multibuild and co + # e.g. obs://build.opensuse.org/openSUSE:Factory/standard/e71360fe635636b65ef2244eb123fc7f-libqt5-qttools + if ($qq{1123}) { + my $disturl = basename($qq{1123}[0]); + $disturl =~ s,^[^-]*-,,; + $source[0] = $disturl; + } + $out .= "=Src: $source[0] $source[1] $source[2] src\n"; + foreach my $bname (@basenames) { my $mode = shift @modes; my $di = shift @dirindexes; @@ -96,21 +114,20 @@ my $filename = $dirs[$di] . $bname; my $fs = $filename; - if ( Fcntl::S_ISLNK($mode) ) { + if (Fcntl::S_ISLNK($mode)) { $fs = "$filename -> $linkto"; } $out .= sprintf "%o %o %s:%s %s\n", $mode, $flag, $user, $group, $fs; - if ( $filename =~ /^\/etc\// + if ($filename =~ /^\/etc\// || $filename =~ /bin\// - || $filename eq "/usr/lib/sendmail" ) + || $filename eq "/usr/lib/sendmail") { push @xprvs, $filename; } - } $out .= "-Flx:\n"; $out .= "+Prv:\n"; - foreach my $prv ( @{ $qq{1047} || [] } ) { + foreach my $prv (@{$qq{1047} || []}) { $out .= "$prv\n"; } foreach my $prv (@xprvs) { @@ -118,30 +135,30 @@ } $out .= "-Prv:\n"; $out .= "+Con:\n"; - foreach my $prv ( @{ $qq{1054} || [] } ) { + foreach my $prv (@{$qq{1054} || []}) { $out .= "$prv\n"; } $out .= "-Con:\n"; $out .= "+Req:\n"; - foreach my $prv ( @{ $qq{1049} || [] } ) { - next if ( $prv eq "this-is-only-for-build-envs" ); + foreach my $prv (@{$qq{1049} || []}) { + next if ($prv eq "this-is-only-for-build-envs"); # Completely disgusting, but maintainers have no interest in fixing, # see #1153 for more details. next - if ( $name =~ "^installation-images-debuginfodeps.*" - && $prv =~ m/debuginfo.build/ ); + if ($name =~ "^installation-images-debuginfodeps.*" + && $prv =~ m/debuginfo.build/); $out .= "$prv\n"; } $out .= "-Req:\n"; $out .= "+Obs:\n"; - foreach my $prv ( @{ $qq{1090} || [] } ) { + foreach my $prv (@{$qq{1090} || []}) { $out .= "$prv\n"; } $out .= "-Obs:\n"; $out .= "+Rec:\n"; - foreach my $prv ( @{ $qq{5046} || [] } ) { + foreach my $prv (@{$qq{5046} || []}) { # ignore boolean dependencies next if $prv =~ m/^\(/; $out .= "$prv\n"; @@ -149,19 +166,19 @@ $out .= "-Rec:\n"; $out .= "+Sup:\n"; - foreach my $prv ( @{ $qq{5052} || [] } ) { + foreach my $prv (@{$qq{5052} || []}) { $out .= "$prv\n"; } $out .= "-Sup:\n"; $out .= "+Enh:\n"; - foreach my $prv ( @{ $qq{5055} || [] } ) { + foreach my $prv (@{$qq{5055} || []}) { $out .= "$prv\n"; } $out .= "-Enh:\n"; $out .= "+Sug:\n"; - foreach my $prv ( @{ $qq{5049} || [] } ) { + foreach my $prv (@{$qq{5049} || []}) { $out .= "$prv\n"; } $out .= "-Sug:\n"; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20190726.df07bcc2/dist/package/openSUSE-release-tools.spec new/openSUSE-release-tools-20190729.70133114/dist/package/openSUSE-release-tools.spec --- old/openSUSE-release-tools-20190726.df07bcc2/dist/package/openSUSE-release-tools.spec 2019-07-26 13:15:55.000000000 +0200 +++ new/openSUSE-release-tools-20190729.70133114/dist/package/openSUSE-release-tools.spec 2019-07-29 10:19:25.000000000 +0200 @@ -426,7 +426,6 @@ %{_bindir}/osrt-legal-auto %{_bindir}/osrt-obs_clone %{_bindir}/osrt-openqa-maintenance -%{_bindir}/osrt-rebuildpacs %{_bindir}/osrt-requestfinder %{_bindir}/osrt-scan_baselibs %{_bindir}/osrt-status diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20190726.df07bcc2/gocd/rebuild-trigger.gocd.yaml new/openSUSE-release-tools-20190729.70133114/gocd/rebuild-trigger.gocd.yaml --- old/openSUSE-release-tools-20190726.df07bcc2/gocd/rebuild-trigger.gocd.yaml 2019-07-26 13:15:55.000000000 +0200 +++ new/openSUSE-release-tools-20190729.70133114/gocd/rebuild-trigger.gocd.yaml 2019-07-29 10:19:25.000000000 +0200 @@ -18,26 +18,23 @@ - repo-checker tasks: - script: |- - ln -sf /home/go/config/rebuild.problems problems - ln -sf /home/go/config/rebuild.buildinfos buildinfos - - script: |- echo "openSUSE:Factory" - ./rebuildpacs.pl openSUSE:Factory standard i586 - ./rebuildpacs.pl openSUSE:Factory standard x86_64 + ./project-installcheck.py --osc-debug --debug rebuild --store openSUSE:Factory:Staging/dashboard openSUSE:Factory standard i586 + ./project-installcheck.py --osc-debug --debug rebuild --store openSUSE:Factory:Staging/dashboard openSUSE:Factory standard x86_64 - script: |- echo "openSUSE:Factory:PowerPC" - ./rebuildpacs.pl openSUSE:Factory:PowerPC standard ppc - ./rebuildpacs.pl openSUSE:Factory:PowerPC standard ppc64 - ./rebuildpacs.pl openSUSE:Factory:PowerPC standard ppc64le + ./project-installcheck.py --osc-debug --debug rebuild --store openSUSE:Factory:Staging/dashboard openSUSE:Factory:PowerPC standard ppc + ./project-installcheck.py --osc-debug --debug rebuild --store openSUSE:Factory:Staging/dashboard openSUSE:Factory:PowerPC standard ppc64 + ./project-installcheck.py --osc-debug --debug rebuild --store openSUSE:Factory:Staging/dashboard openSUSE:Factory:PowerPC standard ppc64le - script: |- echo "openSUSE Leap 15.2" - echo ./rebuildpacs.pl openSUSE:Leap:15.2 standard i586 - echo ./rebuildpacs.pl openSUSE:Leap:15.2 standard x86_64 + ./project-installcheck.py --osc-debug --debug rebuild --store openSUSE:Leap:15.2:Staging/dashboard openSUSE:Leap:15.2 standard i586 + ./project-installcheck.py --osc-debug --debug rebuild --store openSUSE:Leap:15.2:Staging/dashboard openSUSE:Leap:15.2 standard x86_64 - script: |- echo "GNOME devel projects" - ./rebuildpacs.pl GNOME:Factory openSUSE_Factory i586 - ./rebuildpacs.pl GNOME:Factory openSUSE_Factory x86_64 - ./rebuildpacs.pl GNOME:Factory openSUSE_PPC ppc64le - ./rebuildpacs.pl GNOME:Next openSUSE_Factory i586 - ./rebuildpacs.pl GNOME:Next openSUSE_Factory x86_64 - ./rebuildpacs.pl GNOME:Next openSUSE_PPC ppc64le + ./project-installcheck.py --osc-debug --debug rebuild --store home:repo-checker/rebuilds GNOME:Factory openSUSE_Factory i586 + ./project-installcheck.py --osc-debug --debug rebuild --store home:repo-checker/rebuilds GNOME:Factory openSUSE_Factory x86_64 + ./project-installcheck.py --osc-debug --debug rebuild --store home:repo-checker/rebuilds GNOME:Factory openSUSE_PPC ppc64le + ./project-installcheck.py --osc-debug --debug rebuild --store home:repo-checker/rebuilds GNOME:Next openSUSE_Factory i586 + ./project-installcheck.py --osc-debug --debug rebuild --store home:repo-checker/rebuilds GNOME:Next openSUSE_Factory x86_64 + ./project-installcheck.py --osc-debug --debug rebuild --store home:repo-checker/rebuilds GNOME:Next openSUSE_PPC ppc64le diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20190726.df07bcc2/osclib/repochecks.py new/openSUSE-release-tools-20190729.70133114/osclib/repochecks.py --- old/openSUSE-release-tools-20190726.df07bcc2/osclib/repochecks.py 2019-07-26 13:15:55.000000000 +0200 +++ new/openSUSE-release-tools-20190729.70133114/osclib/repochecks.py 2019-07-29 10:19:25.000000000 +0200 @@ -1,10 +1,12 @@ import logging -import tempfile import os import re -import yaml import subprocess +import tempfile from fnmatch import fnmatch + +import yaml + from osclib.cache_manager import CacheManager logger = logging.getLogger('InstallChecker') @@ -12,13 +14,17 @@ SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) CACHEDIR = CacheManager.directory('repository-meta') + class CorruptRepos(Exception): pass # the content of sp is name, version, release, arch + + def _format_pkg(sp): return "{}-{}-{}.{}".format(sp[0], sp[1], sp[2], sp[3]) + def _check_exists_in_whitelist(sp, whitelist): if sp[0] in whitelist: logger.debug("Found %s in whitelist, ignoring", sp[0]) @@ -33,11 +39,13 @@ logger.debug("Found %s matching whitelist entry %s, ignoring", sp[0], entry) return True + def _check_colon_format(sp1, sp2, whitelist): if "{}:{}".format(sp1, sp2) in whitelist: logger.debug("Found %s:%s in whitelist, ignoring", sp1, sp2) return True + def _check_conflicts_whitelist(sp1, sp2, whitelist): if _check_exists_in_whitelist(sp1, whitelist): return True @@ -48,6 +56,7 @@ if _check_colon_format(sp2[0], sp1[0], whitelist): return True + def _fileconflicts(pfile, target_packages, whitelist): script = os.path.join(SCRIPT_PATH, '..', 'findfileconflicts') p = subprocess.run(['perl', script, pfile], stdout=subprocess.PIPE) @@ -72,15 +81,18 @@ if len(output): return output -def _installcheck(pfile, arch, target_packages, whitelist): + +def parsed_installcheck(pfile, arch, target_packages, whitelist): + reported_problems = dict() + if not len(target_packages): - return None + return reported_problems p = subprocess.run(['/usr/bin/installcheck', arch, pfile], stdout=subprocess.PIPE, text=True) if p.returncode: - output = '' in_problem = False - install_re = re.compile(r"^can't install (.*)-[^-]+-[^-]+:$") + package = None + install_re = re.compile(r"^can't install (.*)(-[^-]+-[^-]+):$") for line in p.stdout.split('\n'): if not line.startswith(' '): in_problem = False @@ -93,10 +105,14 @@ if package in whitelist: logger.debug("{} fails installcheck but is white listed".format(package)) continue + reported_problems[package] = {'problem': match.group(1) + match.group(2), 'output': [], 'source': target_packages[package]} in_problem = True + continue if in_problem: - output += line + "\n" - return output + reported_problems[package]['output'].append(line[2:]) + + return reported_problems + def installcheck(directories, arch, whitelist, ignore_conflicts): @@ -122,16 +138,22 @@ if output: parts.append(output) - output = _installcheck(pfile, arch, target_packages, whitelist) - if output: + parsed = parsed_installcheck(pfile, arch, target_packages, whitelist) + if len(parsed): + output = '' + for package in sorted(parsed): + output += "can't install " + parsed[package]['problem'] + ":\n" + output += "\n".join(parsed[package]['output']) + output += "\n\n" parts.append(output) return parts + def mirror(apiurl, project, repository, arch): """Call bs_mirrorfull script to mirror packages.""" directory = os.path.join(CACHEDIR, project, repository, arch) - #return directory + # return directory if not os.path.exists(directory): os.makedirs(directory) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20190726.df07bcc2/project-installcheck.py new/openSUSE-release-tools-20190729.70133114/project-installcheck.py --- old/openSUSE-release-tools-20190726.df07bcc2/project-installcheck.py 2019-07-26 13:15:55.000000000 +0200 +++ new/openSUSE-release-tools-20190729.70133114/project-installcheck.py 2019-07-29 10:19:25.000000000 +0200 @@ -1,22 +1,39 @@ #!/usr/bin/python3 +import datetime +import difflib +import hashlib import logging import os +import os.path +import re +import subprocess import sys -from collections import namedtuple +import tempfile +import cmdln +from urllib.parse import urlencode +import yaml +from lxml import etree as ET from osc import conf import ToolBase -from osclib.cache_manager import CacheManager from osclib.conf import Config -from osclib.core import (repository_path_expand, repository_path_search, - target_archs, project_pseudometa_file_ensure) -from osclib.repochecks import mirror, installcheck +from osclib.core import (http_GET, http_POST, makeurl, + project_pseudometa_file_ensure, + repository_path_expand, repository_path_search, + target_archs, source_file_load, source_file_ensure) +from osclib.repochecks import installcheck, mirror, parsed_installcheck, CorruptRepos class RepoChecker(): def __init__(self): - self.logger = logging.getLogger(__name__) + self.logger = logging.getLogger('RepoChecker') + self.store_project = None + self.store_package = None + + def parse_store(self, project_package): + if project_package: + self.store_project, self.store_package = project_package.split('/') def project_only(self, project): repository = self.project_repository(project) @@ -80,6 +97,23 @@ return result + def _split_and_filter(self, output): + output = output.split("\n") + rebuild_counter_re = re.compile(r'(needed by [^ ]*\-[^-]*)\-[^-]*\.\w+$') + for lnr, line in enumerate(output): + if line.startswith('FOLLOWUP'): + # there can be multiple lines with missing providers + while lnr >= 0 and output[lnr - 1].endswith('none of the providers can be installed'): + output.pop() + lnr = lnr - 1 + for lnr in reversed(range(len(output))): + # those lines are hardly interesting for us + if output[lnr].find('(we have') >= 0: + del output[lnr] + else: + output[lnr] = rebuild_counter_re.sub(r'\1', output[lnr]) + return output + def project_repository(self, project): repository = Config.get(self.apiurl, project).get('main-repo') if not repository: @@ -97,6 +131,173 @@ return repository + def store_yaml(self, state): + state_yaml = yaml.dump(state, default_flow_style=False) + source_file_ensure(self.apiurl, self.store_project, self.store_package, + self.store_filename, state_yaml, comment='Updated rebuild infos') + + def rebuild(self, project, repository, arch): + config = Config.get(self.apiurl, project) + + oldstate = None + self.store_filename = 'rebuildpacs.{}-{}.yaml'.format(project, repository) + state_yaml = source_file_load(self.apiurl, self.store_project, self.store_package, + self.store_filename) + if state_yaml: + oldstate = yaml.safe_load(state_yaml) + + oldstate = oldstate or {} + oldstate.setdefault('check', {}) + oldstate.setdefault('leafs', {}) + + repository_pairs = repository_path_expand(self.apiurl, project, repository) + directories = [] + for pair_project, pair_repository in repository_pairs: + directories.append(mirror(self.apiurl, pair_project, pair_repository, arch)) + + parsed = dict() + with tempfile.TemporaryDirectory(prefix='repochecker') as dir: + pfile = os.path.join(dir, 'packages') + + SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) + script = os.path.join(SCRIPT_PATH, 'write_repo_susetags_file.pl') + parts = ['perl', script, dir] + directories + + p = subprocess.run(parts) + if p.returncode: + # technically only 126, but there is no other value atm - + # so if some other perl error happens, we don't continue + raise CorruptRepos + + target_packages = [] + with open(os.path.join(dir, 'catalog.yml')) as file: + catalog = yaml.safe_load(file) + target_packages = catalog.get(directories[0], []) + + parsed = parsed_installcheck(pfile, arch, target_packages, []) + for package in parsed: + parsed[package]['output'] = "\n".join(parsed[package]['output']) + + # let's risk a N*N algorithm in the hope that we have a limited N + for package1 in parsed: + output = parsed[package1]['output'] + for package2 in parsed: + if package1 == package2: + continue + output = output.replace(parsed[package2]['output'], 'FOLLOWUP(' + package2 + ')') + parsed[package1]['output'] = output + + for package in parsed: + parsed[package]['output'] = self._split_and_filter(parsed[package]['output']) + + url = makeurl(self.apiurl, ['build', project, '_result'], { + 'repository': repository, 'arch': arch, 'code': 'succeeded'}) + root = ET.parse(http_GET(url)).getroot() + succeeding = list(map(lambda x: x.get('package'), root.findall('.//status'))) + + per_source = dict() + + for package, entry in parsed.items(): + source = "{}/{}/{}/{}".format(project, repository, arch, entry['source']) + per_source.setdefault(source, {'output': [], 'builds': entry['source'] in succeeding}) + per_source[source]['output'].extend(entry['output']) + + rebuilds = set() + + for source in sorted(per_source): + if not len(per_source[source]['output']): + continue + self.logger.debug("{} builds: {}".format(source, per_source[source]['builds'])) + self.logger.debug(" " + "\n ".join(per_source[source]['output'])) + if not per_source[source]['builds']: # nothing we can do + continue + old_output = oldstate['check'].get(source, {}).get('problem', []) + if old_output == per_source[source]['output']: + self.logger.debug("unchanged problem") + continue + self.logger.info("rebuild %s", source) + rebuilds.add(os.path.basename(source)) + for line in difflib.unified_diff(old_output, per_source[source]['output'], 'before', 'now'): + self.logger.debug(line.strip()) + oldstate['check'][source] = {'problem': per_source[source]['output'], + 'rebuild': str(datetime.datetime.now())} + + for source in list(oldstate['check']): + if not source.startswith('{}/{}/{}/'.format(project, repository, arch)): + continue + if not os.path.basename(source) in succeeding: + continue + if source not in per_source: + self.logger.info("No known problem, erasing %s", source) + del oldstate['check'][source] + + packages = config.get('rebuildpacs-leafs', '').split() + + # first round: collect all infos from obs + infos = dict() + for package in packages: + subpacks, build_deps = self.check_leaf_package(project, repository, arch, package) + infos[package] = {'subpacks': subpacks, 'deps': build_deps} + + # calculate rebuild triggers + rebuild_triggers = dict() + for package1 in packages: + for package2 in packages: + if package1 == package2: + continue + for subpack in infos[package1]['subpacks']: + if subpack in infos[package2]['deps']: + rebuild_triggers.setdefault(package1, set()) + rebuild_triggers[package1].add(package2) + # ignore this depencency. we already trigger both of them + del infos[package2]['deps'][subpack] + + # calculate build info hashes + for package in packages: + if not package in succeeding: + self.logger.debug("Ignore %s for the moment, not succeeding", package) + continue + m = hashlib.sha256() + for bdep in sorted(infos[package]['deps']): + m.update(bytes(bdep + '-' + infos[package]['deps'][bdep], 'utf-8')) + state_key = '{}/{}/{}/{}'.format(project, repository, arch, package) + olddigest = oldstate['leafs'].get(state_key, {}).get('buildinfo') + if olddigest == m.hexdigest(): + continue + self.logger.info("rebuild leaf package %s (%s vs %s)", package, olddigest, m.hexdigest()) + rebuilds.add(package) + oldstate['leafs'][state_key] = {'buildinfo': m.hexdigest(), + 'rebuild': str(datetime.datetime.now())} + + if self.dryrun: + self.logger.info("To rebuild: %s", ' '.join(rebuilds)) + return + + if not len(rebuilds): + self.logger.debug("Nothing to rebuild") + # in case we do rebuild, wait for it to succeed before saving + self.store_yaml(oldstate) + return + + query = {'cmd': 'rebuild', 'repository': repository, 'arch': arch, 'package': rebuilds} + url = makeurl(self.apiurl, ['build', project], urlencode(query, doseq=True)) + http_POST(url) + + self.store_yaml(oldstate) + + def check_leaf_package(self, project, repository, arch, package): + url = makeurl(self.apiurl, ['build', project, repository, arch, package, '_buildinfo']) + root = ET.parse(http_GET(url)).getroot() + subpacks = set() + for sp in root.findall('subpack'): + subpacks.add(sp.text) + build_deps = dict() + for bd in root.findall('bdep'): + if bd.get('notmeta') == '1': + continue + build_deps[bd.get('name')] = bd.get('version') + '-' + bd.get('release') + return subpacks, build_deps + class CommandLineInterface(ToolBase.CommandLineInterface): @@ -104,15 +305,25 @@ ToolBase.CommandLineInterface.__init__(self, args, kwargs) def setup_tool(self): - tool = RepoChecker() - if self.options.debug: - logging.basicConfig(level=logging.DEBUG) - elif self.options.verbose: - logging.basicConfig(level=logging.INFO) + return RepoChecker() - return tool + @cmdln.option('--store', help='Project/Package to store the rebuild infos in') + def do_rebuild(self, subcmd, opts, project, repository, arch): + """${cmd_name}: Rebuild packages in rebuild=local projects + + ${cmd_usage} + ${cmd_option_list} + """ + self.tool.parse_store(opts.store) + self.tool.apiurl = conf.config['apiurl'] + self.tool.rebuild(project, repository, arch) def do_project_only(self, subcmd, opts, project): + """${cmd_name}: Update repository repo of a project + + ${cmd_usage} + ${cmd_option_list} + """ self.tool.apiurl = conf.config['apiurl'] self.tool.project_only(project) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20190726.df07bcc2/rebuildpacs.pl new/openSUSE-release-tools-20190729.70133114/rebuildpacs.pl --- old/openSUSE-release-tools-20190726.df07bcc2/rebuildpacs.pl 2019-07-26 13:15:55.000000000 +0200 +++ new/openSUSE-release-tools-20190729.70133114/rebuildpacs.pl 1970-01-01 01:00:00.000000000 +0100 @@ -1,398 +0,0 @@ -#! /usr/bin/perl - -use Data::Dumper; -use XML::Simple; -use URI::Escape; -use File::Basename; -use File::Temp qw/tempdir/; -use Digest::MD5 qw(md5_hex); - -my $script_dir; - -BEGIN { - ($script_dir) = $0 =~ m-(.*)/-; - $script_dir ||= '.'; - unshift @INC, $script_dir; -} - -require CreatePackageDescr; - -my @repodirs; - -sub find_source_container($) { - my $pkg = shift; - - for my $repodir (@repodirs) { - my @rpms = glob("$repodir/*-$pkg.rpm"); - for my $rpm (@rpms) { - - # 1123 == Disturl - my %qq = Build::Rpm::rpmq( $rpm, qw{NAME 1123} ); - next if ( $qq{NAME}[0] ne $pkg ); - my $distfile = basename( $qq{1123}[0] ); - $distfile =~ s,^[^-]*-,,; - - return $distfile; - } - } -} - -my $followup = 0; -my $cproblem = ''; -my %problems; - -my $project = $ARGV[0] || "openSUSE:Factory"; -my $repo = $ARGV[1] || "standard"; -my $arch = $ARGV[2] || "x86_64"; - -my %leafed; - -sub read_plain_index($) { - my $file = shift; - - my %ret; - - open( FILE, $file ) || return \%ret; - while (<FILE>) { - if (m/^(.*):(.*)/) { - $ret{$1} = $2; - } - } - close(FILE); - return \%ret; -} - -sub write_plain_index($$) { - my $file = shift; - my $hash = shift; - - open( FILE, ">$file" ) || die "can't write to $file"; - for my $key ( sort keys %{$hash} ) { - print FILE "$key:" . $hash->{$key} . "\n"; - } - close(FILE); -} - -# defines packages that need to be triggered too -my %parents = ( - "rpmlint" => [qw(rpmlint-mini)], - "branding-openSUSE" => [ - qw(glib2-branding-openSUSE - kiwi-config-openSUSE - xfce4-branding-openSUSE - kdebase4-openSUSE kde-branding-openSUSE - bundle-lang-kde bundle-lang-common - installation-images:openSUSE) - ], - "kdebase4-openSUSE" => [qw(bundle-lang-kde)], - "kernel-source" => [qw(perf)], -); - -# for subsets (staging projects) we need to remember which are ignored -my %ignored; - -sub check_leaf_package($$) { - - my $package = shift; - my $rebuildhash = shift; - - my @lines = (); - open( OSC, - "osc api /build/$project/$repo/$arch/$package/_buildinfo|" ); - while (<OSC>) { - chomp; - if (m/<subpack>(.*)</) { - $leafed{$1} = $package; - } - if (m/bdep name="([^"]*)"/) { - my $parent = $leafed{$1}; - if ( $parent && $parent ne "rpmlint-mini" ) { - - # I dislike grep - unless ( grep { $_ eq $package } @{ $parents{$parent} } ) { - print "ADD $package to PARENT $parent!!\n"; - } - next; - } - } - else { - next; - } - next if (m/notmeta="1"/); - push( @lines, $_ ); - } - close(OSC); - my $ctx = Digest::MD5->new; - for my $line ( sort @lines ) { - $ctx->add($line); - } - my $rebuilds = read_plain_index("buildinfos"); - my $newmd5 = $ctx->hexdigest; - if ( $rebuilds->{"$project/$repo/$arch/$package"} ne $newmd5 ) { - - $rebuildhash->{$package} = 1; - for my $child ( @{ $parents{$package} } ) { - $rebuildhash->{$child} = 1; - } - $rebuilds->{"$project/$repo/$arch/$package"} = $newmd5; - write_plain_index( "buildinfos", $rebuilds ); - } -} - -my %torebuild; -check_leaf_package( "rpmlint-mini", \%torebuild ); -check_leaf_package( "rpmlint", \%torebuild ); - -check_leaf_package( "branding-openSUSE", \%torebuild ); -check_leaf_package( "PackageKit-branding-openSUSE", \%torebuild ); -check_leaf_package( "xfce4-branding-openSUSE", \%torebuild ); - -check_leaf_package( "installation-images:openSUSE", \%torebuild ); -check_leaf_package( "installation-images:Kubic", \%torebuild ); -check_leaf_package( "installation-images-extras", \%torebuild ); - -if (%torebuild) { - my $api = "/build/$project?cmd=rebuild&repository=$repo&arch=$arch"; - for my $package ( sort keys %torebuild ) { - next if ( defined $ignored{$package} ); - last if ( length($api) > 32767 ); - $api .= "&package=" . uri_escape($package); - } - system("osc api -X POST '$api'"); -} - -my $pfile = - tempdir( CLEANUP => 1 ) . "/packages"; # the filename is important ;( - -sub mirror_repo($$$) { - - my $project = shift; - my $repo = shift; - my $arch = shift; - - # Old and new in single directory, but never deployed together. - my $repodir = ( $ENV{XDG_CACHE_HOME} || $ENV{HOME} . "/.cache" ) - . "/openSUSE-release-tools/repository-meta/repo-$project-$repo-$arch"; - mkdir($repodir); - - system( -"$script_dir/bs_mirrorfull --nodebug https://api.opensuse.org/public/build/$project/$repo/$arch/ $repodir" - ); - return $repodir; -} - -sub find_package_in_project($) { - my $project = shift; - - open( OSC, "osc api /source/$project?expand=1 |" ); - my $xml = XMLin( join( '', <OSC> ), ForceArray => 1 ); - close(OSC); - my @packs = keys %{ $xml->{entry} }; - return shift @packs; -} - -# find a random package - -sub get_paths($$$) { - my $project = shift; - my $repo = shift; - my $arch = shift; - - my $package = find_package_in_project($project); - - open( OSC, "osc api /build/$project/$repo/$arch/$package/_buildinfo|" ); - my $xml = join( '', <OSC> ); - if ( $xml !~ m/^</ ) { - die "failed to open /build/$project/$repo/$arch/$package/_buildinfo"; - } - $xml = XMLin( $xml, ForceArray => 1 ); - close(OSC); - - return $xml->{path}; -} - -my $paths = get_paths( $project, $repo, $arch ); -my @rpms; - -for my $path (@$paths) { - - # openSUSE:Factory/ports is in the paths, but not a repo - if ( - system( -"osc api /build/$path->{'project'}/$path->{'repository'}/$arch > /dev/null 2>&1 " - ) - ) - { - next; - } - - my $repodir = - mirror_repo( $path->{'project'}, $path->{'repository'}, $arch ); - push( @repodirs, $repodir ); - push( @rpms, glob("$repodir/*.rpm") ); -} - -open( PACKAGES, ">", $pfile ) || die "can not open $pfile"; -print PACKAGES "=Ver: 2.0\n"; - -my %knipser; - -foreach my $package (@rpms) { - die $package unless $package =~ m,/.{32}-([^/]+)\.rpm$,; - next if $knipser{$1}++; - my $out = CreatePackageDescr::package_snippet($package); - print PACKAGES $out; -} -close(PACKAGES); - -# read the problems out of installcheck -my $rpmarch = $arch; -$rpmarch = "armv7hl" if ( $arch eq "armv7l" ); -$rpmarch = "armv6hl" if ( $arch eq "armv6l" ); - -open( INSTALLCHECK, "/usr/bin/installcheck $rpmarch $pfile|" ); -while (<INSTALLCHECK>) { - chomp; - - if (m/^can't install (.*)\-[^-]*\-[^-]*\.($rpmarch|noarch):/) { - $cproblem = $1; - $cproblem =~ s/kmp-([^-]*)/kmp-default/; - $cproblem = find_source_container($cproblem); - $followup = 0; - next; - } - - $followup = 1 if ( $_ =~ m/none of the providers can be installed/ ); - - # not interesting for me - next if (m/ \(we have /); - next if ($followup); - - # very thin ice here - s,\(\)\(64bit\),,; - - s,(needed by [^ ]*)\-[^-]*\-[^-]*\.($rpmarch|noarch)$,$1,; - - s,^\s*,,; - - # patterns are too spammy and rebuilding doesn't help - next - if ( - grep { $_ eq $cproblem } - qw( - fftw3:gnu-openmpi-hpc - hdf5:gnu-hpc - hdf5:gnu-mpich-hpc - hdf5:gnu-mvapich2-hpc - hdf5:gnu-openmpi-hpc - hdf5:gnu-openmpi2-hpc - hdf5:gnu-openmpi3-hpc - hdf5:mvapich2 - hdf5:openmpi - hdf5:serial - installation-images:Kubic - metis:gnu-hpc - netcdf:gnu-hpc - netcdf:gnu-mvapich2-hpc - netcdf:gnu-openmpi-hpc - netcdf:openmpi - netcdf:serial - patterns-base - patterns-haskell - patterns-mate - patterns-media - patterns-openSUSE - patterns-yast - petsc:serial - petsc:openmpi - python-numpy:gnu-hpc - scalapack:gnu-mvapich2-hpc - scalapack:gnu-openmpi-hpc - warewulf:modules - python-scipy:gnu-hpc - ) - ); - $problems{$cproblem}->{$_} = 1; - -} -close(INSTALLCHECK); -unlink($pfile); -rmdir( dirname($pfile) ); - -for my $package ( sort keys %problems ) { - $problems{$package} = join( ', ', sort( keys %{ $problems{$package} } ) ); -} - -my @other_problems; -my %oproblems; - -open( PROBLEMS, "problems" ); -while (<PROBLEMS>) { - chomp; - if (m,^$project/$repo/$arch/([^:]*):\s*(.*)$,) { - my $package = $1; - my $oproblem = $2; - - # remember old problems for current project/repo - $oproblems{$package} = $oproblem; - } - else { - # keep all lines for other projects/repos as they are - push @other_problems, $_; - } -} -close(PROBLEMS); - -exit(0) if ( !%problems ); - -# check for succeeded packages - we can't filter as we don't know if the problems are all in the top project ;( -my $api = "/build/$project/_result?repository=$repo&arch=$arch&code=succeeded"; - -open( RESULT, "osc api '$api'|" ); -@result = <RESULT>; -my $results = XMLin( join( '', @result ), ForceArray => ['status'] ); -close(RESULT); - -my @packages = @{ $results->{result}->{status} }; -my $rebuildit = 0; - -$api = "/build/$project?cmd=rebuild&repository=$repo&arch=$arch"; -for my $package (@packages) { - $package = $package->{package}; - last if ( length($api) > 32767 ); - - if ( !$problems{$package} ) { - - # it can go - delete $oproblems{$package}; - next; - } - - my $oproblem = $oproblems{$package} || ''; - if ( $problems{$package} eq $oproblem ) { - - # rebuild won't help - next; - } - $rebuildit = 1; - print "rebuild ", $package, ": ", $problems{$package}, "\n"; - $api .= "&package=" . uri_escape($package); - $oproblems{$package} = $problems{$package}; -} - -open( PROBLEMS, ">problems" ); - -# write all lines for other projects/repos as they are -foreach (@other_problems) { - print PROBLEMS $_, "\n"; -} -for my $package ( keys %oproblems ) { - print PROBLEMS "$project/$repo/$arch/" . $package . ": " - . $oproblems{$package}, "\n"; -} -close(PROBLEMS); - -if ($rebuildit) { - print "API '$api'\n"; - system("osc api -X POST '$api'"); -} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20190726.df07bcc2/write_repo_susetags_file.pl new/openSUSE-release-tools-20190729.70133114/write_repo_susetags_file.pl --- old/openSUSE-release-tools-20190726.df07bcc2/write_repo_susetags_file.pl 2019-07-26 13:15:55.000000000 +0200 +++ new/openSUSE-release-tools-20190729.70133114/write_repo_susetags_file.pl 2019-07-29 10:19:25.000000000 +0200 @@ -1,17 +1,12 @@ #! /usr/bin/perl -w -use File::Basename; -use File::Temp qw/ tempdir /; -use XML::Simple; -use Data::Dumper; -use Cwd; - use strict; +use File::Basename; BEGIN { - my ($wd) = $0 =~ m-(.*)/- ; - $wd ||= '.'; - unshift @INC, $wd; + my ($wd) = $0 =~ m-(.*)/-; + $wd ||= '.'; + unshift @INC, $wd; } require CreatePackageDescr; @@ -19,19 +14,19 @@ die "Usage: $0 <output directory> <input directories...>" if scalar(@ARGV) < 2; my $output_directory = shift @ARGV; -my @directories = @ARGV; +my @directories = @ARGV; sub write_package { - my ($package, $packages_fd, $directory, $written_names) = @_; + my ($package, $packages_fd, $directory, $written_names, $sources) = @_; my $name = basename($package); - if ($name =~ m/^[a-z0-9]{32}-/) { # repo cache - $name =~ s,^[^-]+-(.*)\.rpm,$1,; + if ($name =~ m/^[a-z0-9]{32}-/) { # repo cache + $name =~ s,^[^-]+-(.*)\.rpm,$1,; } else { - $name =~ s,^(.*)-[^-]+-[^-]+.rpm,$1,; + $name =~ s,^(.*)-[^-]+-[^-]+.rpm,$1,; } - if ( defined $written_names->{$name} ) { + if (defined $written_names->{$name}) { return; } $written_names->{$name} = $directory; @@ -41,18 +36,22 @@ print STDERR "ERROR: empty package snippet for: $name\n"; exit(126); } + if ($out =~ m/=Src: ([^ ]*)/) { + $sources->{$name} = $1; + } print $packages_fd $out; return $name; } -open( my $packages_fd, ">", "$output_directory/packages" ) || die 'can not open'; +open(my $packages_fd, ">", "$output_directory/packages") || die 'can not open'; print $packages_fd "=Ver: 2.0\n"; my %written_names; +my %sources; for my $directory (@directories) { my @rpms = glob("$directory/*.rpm"); - write_package( $_, $packages_fd, $directory, \%written_names ) for @rpms; + write_package($_, $packages_fd, $directory, \%written_names, \%sources) for @rpms; } close($packages_fd); @@ -60,16 +59,17 @@ # turn around key->value my %per_directory; for my $name (keys %written_names) { - $per_directory{$written_names{$name}} ||= []; - push(@{$per_directory{$written_names{$name}}}, $name); + $per_directory{$written_names{$name}} ||= []; + push(@{$per_directory{$written_names{$name}}}, $name); } -open( my $yaml_fd, ">", "$output_directory/catalog.yml" ) || die 'can not open'; +open(my $yaml_fd, ">", "$output_directory/catalog.yml") || die 'can not open'; for my $directory (@directories) { - next unless defined($per_directory{$directory}); - print $yaml_fd "$directory:\n"; - for my $name (@{$per_directory{$directory}}) { - print $yaml_fd " - '$name'\n"; - } + next unless defined($per_directory{$directory}); + print $yaml_fd "$directory:\n"; + for my $name (@{$per_directory{$directory}}) { + my $source = $sources{$name} || 'unknown'; + print $yaml_fd " '$name': '$source'\n"; + } } close($yaml_fd); ++++++ openSUSE-release-tools.obsinfo ++++++ --- /var/tmp/diff_new_pack.2amClQ/_old 2019-07-29 17:31:28.730165919 +0200 +++ /var/tmp/diff_new_pack.2amClQ/_new 2019-07-29 17:31:28.730165919 +0200 @@ -1,5 +1,5 @@ name: openSUSE-release-tools -version: 20190726.df07bcc2 -mtime: 1564139755 -commit: df07bcc26288465b421772cd8fa76becb9879156 +version: 20190729.70133114 +mtime: 1564388365 +commit: 701331142b98affdd7eeda304ef8a6b7d8596b2a