Mailinglist Archive: opensuse-buildservice (73 mails)

< Previous Next >
[opensuse-buildservice] OBS scheduler features

I have two patches for review:
1. Fix a problem with different names - OBS package name and RPM name
described in the spec-file.
For example: OBS package name 'linaro-gcc', but name of rpm is 'gcc'.
2. Change a scheduler mechanism. The build of non-cyclic packets wait
the builds of cyclic packets.
This has some advantages: reduce time of project build, reducing number
of packages rebuilds.
I tested the patches on the project with 129 packages (41 cyclic) - the
build time was reduced by ~33%.
I would like to know your opinion about the patches.

--
Best regards,
Oleg Ogurtsov.

Samsung R&D Institute Russia,
e-mail: o.ogurtsov@xxxxxxxxxxx
From 14d4e906cae44701d0fa967d7c4334957bfbf619 Mon Sep 17 00:00:00 2001
From: Pavel Kopyl <p.kopyl@xxxxxxxxxxx>
Date: Mon, 26 Jun 2017 16:59:16 +0300
Subject: [PATCH 1/2] [backend] The different names support for OBS and rpm
packages in the scheduler

---
src/backend/BSSched/BuildJob/Package.pm | 39 +++++++++++++++++++++++++++++++--
src/backend/BSSched/Checker.pm | 28 ++++++++++++++++++++++-
2 files changed, 64 insertions(+), 3 deletions(-)

diff --git a/src/backend/BSSched/BuildJob/Package.pm
b/src/backend/BSSched/BuildJob/Package.pm
index aa0ac8a..cefae40 100644
--- a/src/backend/BSSched/BuildJob/Package.pm
+++ b/src/backend/BSSched/BuildJob/Package.pm
@@ -78,6 +78,8 @@ sub check {
my $prp = $ctx->{'prp'};
my $notready = $ctx->{'notready'};
my $dep2src = $ctx->{'dep2src'};
+ my $src2pkg = $ctx->{'src2pkg'};
+ my $pkg2src = $ctx->{'pkg2src'};
my $edeps = $info->{'edeps'} || $ctx->{'edeps'}->{$packid} || [];
my $depislocal = $ctx->{'depislocal'};
my $gdst = $ctx->{'gdst'};
@@ -98,7 +100,26 @@ sub check {
}

# calculate if we're blocked
- my @blocked = grep {$notready->{$dep2src->{$_}}} @$edeps;
+ # @blocked contain rpm-packages
+ my @blocked;
+
+ # The original code. It's can be restored for upstream logic
+# @blocked = grep {$notready->{$dep2src->{$_}}} @$edeps;
+ # iterate for rpm-packages
+ for my $dep (@$edeps) {
+ my $depsrc = $dep2src->{$dep};
+ print " no depsrc for rpm-package '$dep'\n" unless (defined
$depsrc);
+ my $obs_package = $src2pkg->{$depsrc};
+ # obs package may be undefined if it is present at the parent project
+ #print " no obs_package for rpm-name '$depsrc'\n" unless (defined
$obs_package);
+
+ # notready contain rpm-names or obs-packages
+ if ($notready->{$depsrc} or (defined $obs_package &&
$notready->{$obs_package})) {
+ # need to push rpm-package
+ push @blocked, $dep unless (grep ($dep eq $_, @blocked));
+ }
+ }
+
@blocked = () if $repo->{'block'} && $repo->{'block'} eq 'never';
if ($ctx->{'cychash'}->{$packid}) {
# package belongs to a cycle, prune blocked list
@@ -115,7 +136,21 @@ sub check {
my %cycs = map {$_ => 1} @{$ctx->{'cychash'}->{$packid}};
# prune building cycle packages from blocked
my $building = $ctx->{'building'};
- @blocked = grep {!$cycs{$dep2src->{$_}} || !$building->{$dep2src->{$_}}}
@blocked;
+
+ # @blocked = grep {!$cycs{$src2pkg->{$dep2src->{$_}}} ||
!$building->{$dep2src->{$_}}} @blocked;
+ my @blocked_prune;
+ for my $bpkg (@blocked) { # rpm-packages
+ my $pkgsrc = $dep2src->{$bpkg}; # rpm-name
+ my $pkg = $src2pkg->{$pkgsrc}; # obs-package
+ # %cycs and %building contains obs-packages
+ next if ($cycs{$pkg} && $building->{$pkg});
+
+ push @blocked_prune, $bpkg unless (grep ($bpkg eq $_, @blocked_prune));
+ }
+ if ($#blocked_prune != $#blocked && $ctx->{'verbose'}) {
+ print " prune blocked ($blocked[0]...)\n";
+ }
+ @blocked = @blocked_prune;
if (@blocked) {
if ($ctx->{'verbose'}) {
print " - $packid ($buildtype)\n";
diff --git a/src/backend/BSSched/Checker.pm b/src/backend/BSSched/Checker.pm
index c30ad0f..28298c2 100644
--- a/src/backend/BSSched/Checker.pm
+++ b/src/backend/BSSched/Checker.pm
@@ -399,6 +399,10 @@ sub expandandsort {
my %havepatchinfos;
my %pkg2buildtype;

+ my %pkgid2name;
+ my %name2pkgid;
+ my %pkgiddeps;
+
my $subpacks = $ctx->{'subpacks'};

$ctx->{'experrors'} = \%experrors;
@@ -411,6 +415,9 @@ sub expandandsort {
next;
}
my $info = (grep {$_->{'repository'} eq $repoid} @{$pdata->{'info'} ||
[]})[0];
+ $pkgid2name{$packid} = $info->{'name'} || $packid;
+ $name2pkgid{$pkgid2name{$packid}} = $packid;
+
# calculate package type
my $buildtype;
if ($pdata->{'aggregatelist'}) {
@@ -470,21 +477,40 @@ sub expandandsort {
@edeps = @deps;
}
$pdeps{$packid} = \@edeps;
+ $pkgiddeps{$pkgid2name{$packid}} = \@edeps;
+ }
+
+ my %src2pkg;
+ for my $packid (sort keys %pkg2src) {
+ my $n = $pkg2src{$packid} || $packid;
+ $src2pkg{$n} = $packid;
}

$ctx->{'edeps'} = \%pdeps;
$ctx->{'experrors'} = \%experrors;
$ctx->{'pkg2buildtype'} = \%pkg2buildtype;
+ $ctx->{'src2pkg'} = \%src2pkg;
+ $ctx->{'pkg2src'} = \%pkg2src;

+ my @packnames = map {$pkgid2name{$_}} @$packs;
# now sort
print " sorting ".@$packs." packages\n";
my @cycles;
if (@$packs > 1) {
- @$packs = BSSolv::depsort(\%pdeps, $ctx->{'dep2src'}, \@cycles, @$packs);
+ @$packs = BSSolv::depsort(\%pkgiddeps, $ctx->{'dep2src'}, \@cycles,
@packnames);
if (@cycles) {
print "cycle: ".join(' -> ', @$_)."\n" for @cycles;
}
}
+ @$packs = map {$name2pkgid{$_}} @$packs;
+ for my $cyc (@cycles) {
+ if (ref($cyc) eq 'ARRAY') {
+ $_ = $name2pkgid{$_} for @{$cyc};
+ } else {
+ $cyc = $name2pkgid{$cyc};
+ }
+ }
+
if (%havepatchinfos) {
# bring patchinfos to back
my @packs_patchinfos = grep {$havepatchinfos{$_}} @$packs;
--
2.7.4

From b82f55ee95760918300c9032aa45c59b01bcaa1b Mon Sep 17 00:00:00 2001
From: Oleg Ogurtsov <o.ogurtsov@xxxxxxxxxxx>
Date: Mon, 17 Jul 2017 11:40:40 +0300
Subject: [PATCH 2/2] [backend] Optimization for cyclic packages.

Change a scheduler mechanism - waiting of the builds end of cyclic packets.
---
src/backend/BSSched/BuildJob/Package.pm | 44 ++++++++++++++++++++++++++++++---
src/backend/BSSched/Checker.pm | 13 ++++++++++
2 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/src/backend/BSSched/BuildJob/Package.pm
b/src/backend/BSSched/BuildJob/Package.pm
index cefae40..29b93ca 100644
--- a/src/backend/BSSched/BuildJob/Package.pm
+++ b/src/backend/BSSched/BuildJob/Package.pm
@@ -102,9 +102,11 @@ sub check {
# calculate if we're blocked
# @blocked contain rpm-packages
my @blocked;
-
+ my %blocked_obs_optimization;
+ my $subpacks = $ctx->{'subpacks'};
+
# The original code. It's can be restored for upstream logic
-# @blocked = grep {$notready->{$dep2src->{$_}}} @$edeps;
+ # @blocked = grep {$notready->{$dep2src->{$_}}} @$edeps;
# iterate for rpm-packages
for my $dep (@$edeps) {
my $depsrc = $dep2src->{$dep};
@@ -118,6 +120,42 @@ sub check {
# need to push rpm-package
push @blocked, $dep unless (grep ($dep eq $_, @blocked));
}
+
+ next unless $obs_package;
+
+ # ignore already checked obs-package
+ next if ($blocked_obs_optimization{$obs_package});
+ $blocked_obs_optimization{$obs_package} = 1;
+
+ #print "depid: $obs_package\n";
+ my $depcyc = $ctx->{'cychash'}->{$obs_package};
+
+ # package $obs_package doesn't belong to a cycle.
+ next unless $depcyc;
+ #print "depcyc:\n".Dumper($depcyc)."\n";
+
+ # ignore cycle if it conains $packid;
+ next if (grep (/^${packid}$/, @$depcyc) );
+
+ # check cyclic packages (obs-package)
+ for my $cycpid (@$depcyc) {
+ # ignore double pushing $obs_package;
+ next if ($cycpid eq $obs_package);
+
+ my $cycpsrc = $pkg2src->{$cycpid}; # rpm-name
+ next unless ($cycpsrc);
+ next unless ($notready->{$cycpsrc} or $notready->{$cycpid});
+
+ my $current_subpacks = $subpacks->{$cycpsrc};
+ print "FATAL ERROR: no subpacks for '$cycpsrc'\n" unless
($current_subpacks);
+ next unless ($current_subpacks);
+
+ # Need to add rpm-packages to @blocked
+ foreach my $current_subpack (@$current_subpacks) {
+ # push rpm-package if need
+ push @blocked, $current_subpack unless (grep ($current_subpack eq $_,
@blocked));
+ }
+ }
}

@blocked = () if $repo->{'block'} && $repo->{'block'} eq 'never';
@@ -254,7 +292,7 @@ sub check {
my $m = $dep2meta->{$bin};
if (!$m) {
# meta file is not in cache, read it from the full tree
- # the next line works for deb and rpm
+ # the next line works for deb and rpm
my $mf = substr("$reporoot/$path", 0, -4);
if (-e "$mf.meta") {
my @s = stat(_);
diff --git a/src/backend/BSSched/Checker.pm b/src/backend/BSSched/Checker.pm
index 28298c2..367a5f9 100644
--- a/src/backend/BSSched/Checker.pm
+++ b/src/backend/BSSched/Checker.pm
@@ -670,8 +670,21 @@ sub checkpkgs {
}

my @cpacks = @{$ctx->{'packs'}};
+ # for moving noncyclic packages to the end array
+ my %noncyclic;
+ my $cyclic_size = keys %cychash;
while (@cpacks) {
my $packid = shift @cpacks;
+
+ if ($cyclic_size > 0) {
+ unless ($cychash{$packid} || $noncyclic{$packid}) {
+ # Looped packages are processed primarily (for filling a hash
%notready)
+ $noncyclic{$packid} = 1;
+ push @cpacks, $packid;
+ next;
+ }
+ }
+
my $incycle = 0;
if ($cychash{$packid}) {
next if $packstatus{$packid} && $packstatus{$packid} ne 'done'; #
already decided in phase 1
--
2.7.4

< Previous Next >
Follow Ups