Mailinglist Archive: opensuse-buildservice (348 mails)

< Previous Next >
[opensuse-buildservice] new version of the dod patch
Hi,

here's a new version of the dod patch.

Changes:
- bs_repserver:
* hand-off all fetching requests to the ajaxsocket: this means only
one request for the same project/repo/arch is served and all other
subsequent requests for the same project/repo/arch are "blocked" until
the first request finished. This way we can avoid that the same package
is fetched multiple times. The drawback of this approach is that the
worst-case running time is O(n^2) (where n := number of packages to be
fetched) but this can be ignored because it is usually a one-time job
IMHO.
* notify scheduler if a package was fetched
- bs_sched:
* removed old code to support updates of the metafile (now the repserver has
to notify the scheduler if it fetched a package). For instance if a more
frequently changing repo (e.g. an update repo) is used as a dod repo
the scheduler has to detect "meta changes" and trigger rebuilds (which
was not possible with the old code).

Feedback, objections etc. are welcome:)

Btw. I've a general question about the ajaxsocket. For instance if the
srcserver does a RPC to the repserver and the repserver hands-off this request
to its ajaxsocket, should the srcserver hand-off the request to its ajaxsocket
too (before it does the RPC)?
If I understand it correctly all requests which can't be served "immediately"
are handed-off to the ajaxsocket because otherwise they may "block" other
requests which can be served immediately (if the "maxchild" limit is reached).
If this is correct why do we handoff requests like
"/build/<projid>/<repoid>/<arch>/_repository?view=solvstate"?
Can you enlighten me?:)


Marcus
diff --git a/src/backend/BSSolv.xs b/src/backend/BSSolv.xs
index f69f339..5638c7a 100644
--- a/src/backend/BSSolv.xs
+++ b/src/backend/BSSolv.xs
@@ -1442,7 +1442,12 @@ pkg2fullpath(BSSolv::pool pool, int p, char *myarch)
CODE:
{
unsigned int medianr;
- const char *s = solvable_get_location(pool->solvables + p,
&medianr);
+ Solvable *solv = pool->solvables + p;
+ char *s = solvable_get_location(solv, &medianr);
+ char *tmp = strrchr(s, '/');
+ /* dod: ignore relative url path */
+ if (tmp && repo_lookup_str(solv->repo, SOLVID_META,
buildservice_dodurl))
+ s = strcpy(s, tmp + 1);
Repo *repo = pool->solvables[p].repo;
s = pool_tmpjoin(pool, myarch, "/:full/", s);
RETVAL = pool_tmpjoin(pool, repo->name, "/", s);
diff --git a/src/backend/bs_repserver b/src/backend/bs_repserver
index d2fa62e..f0e9e2e 100755
--- a/src/backend/bs_repserver
+++ b/src/backend/bs_repserver
@@ -164,18 +164,30 @@ sub addrepo_remote {
return $pool->repofromdata($prp, $cache);
}

-
-#XXX: DOD
-# if ($cache->{'/url'}) {
-# for my $b (@todo) {
-# next unless $cache->{$b};
-# my $bin = "$prp/$arch/:full/$cache->{$b}->{'path'}";
-# print "fetch: $b\n";
-# fetchbinary("$reporoot/$bin", $cache->{$b}, $cache->{'/url'});
-# next unless -f "$reporoot/$bin";
-# $bins{$b} = $bin;
-# }
-
+sub fetchbinary {
+ my ($pool, $repo, $fullfilename, $bin, $maxredirects) = @_;
+ die($repo->name() . " is no dod repo") unless $repo->dodurl();
+ return $fullfilename if -e $fullfilename;
+ my %rnames = $repo ? $repo->pkgnames() : ();
+ my $p = $rnames{$bin};
+ die("$bin: not available") unless $p;
+ my $url = $repo->dodurl() . "/" . $pool->pkg2path($p);
+ my $tmp = "$fullfilename$$";
+ my $r;
+ eval {
+ #print "fetching: $url\n";
+ my $param = {'uri' => $url, 'filename' => $tmp, 'receiver' =>
\&BSHTTP::file_receiver};
+ $param->{'maxredirects'} = $maxredirects if defined($maxredirects);
+ $r = BSWatcher::rpc($param);
+ };
+ if ($@) {
+ unlink($tmp);
+ die($@);
+ }
+ return unless defined($r);
+ rename($tmp, $fullfilename) || die("rename: $!\n");
+ return $fullfilename;
+}

sub getbinaryversions {
my ($cgi, $projid, $repoid, $arch) = @_;
@@ -185,6 +197,8 @@ sub getbinaryversions {
my $repo = addrepo_scan($pool, $prp, $arch);
my %rnames = $repo ? $repo->pkgnames() : ();
my @res;
+ my $serial = BSWatcher::serialize("$reporoot/$projid/$repoid/$arch") if
$BSStdServer::isajax;
+ return if $BSStdServer::isajax && !defined($serial);
for my $n (@qbins) {
my $p = $rnames{$n};
if (!$p) {
@@ -192,14 +206,29 @@ sub getbinaryversions {
next;
}
my $path = "$reporoot/".$pool->pkg2fullpath($p, $arch);
+ my $sizek = $pool->pkg2sizek($p);
+ my $hdrmd5 = $pool->pkg2pkgid($p);
+ if (!-e $path && $repo->dodurl()) {
+ if (!$BSStdServer::isajax) {
+ BSHandoff::handoff($ajaxsocket, "/getbinaryversions", undef,
"project=$projid", "repository=$repoid", "arch=$arch",
"binaries=$cgi->{'binaries'}");
+ exit(0);
+ }
+ my $r = fetchbinary($pool, $repo, $path, $n, 3);
+ return unless defined($r);
+ }
+ if ($hdrmd5 eq 'd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0') {
+ my @s = stat($path);
+ $sizek = ($s[7] + 1023) >> 10;
+ $hdrmd5 = Build::queryhdrmd5($path);
+ }
my $r;
if ($path =~ /\.rpm$/) {
$r = {'name' => "$n.rpm"};
} else {
$r = {'name' => "$n.deb"};
}
- $r->{'hdrmd5'} = $pool->pkg2pkgid($p);
- $r->{'sizek'} = $pool->pkg2sizek($p);
+ $r->{'hdrmd5'} = $hdrmd5;
+ $r->{'sizek'} = $sizek;
push @res, $r;
next if $cgi->{'nometa'};
next unless $path =~ s/\.(?:rpm|deb)$//;
@@ -212,6 +241,10 @@ sub getbinaryversions {
$r->{'metamd5'} = $ctx->hexdigest();
close F;
}
+ if (defined($serial)) {
+ BSWatcher::serialize_end($serial);
+ forwardevent($cgi, 'scanrepo', $projid, undef, $repoid, $arch);
+ }
undef $repo;
undef $pool;
return ({ 'binary' => \@res }, $BSXML::binaryversionlist);
@@ -226,6 +259,8 @@ sub getbinaries {
my $repo = addrepo_scan($pool, $prp, $arch);
my %rnames = $repo ? $repo->pkgnames() : ();
my @send;
+ my $serial = BSWatcher::serialize("$reporoot/$projid/$repoid/$arch") if
$BSStdServer::isajax;
+ return if $BSStdServer::isajax && !defined($serial);
for my $n (@qbins) {
my $p = $rnames{$n};
if (!$p) {
@@ -233,6 +268,14 @@ sub getbinaries {
next;
}
my $path = "$reporoot/".$pool->pkg2fullpath($p, $arch);
+ if (!-e $path && $repo->dodurl()) {
+ if (!$BSStdServer::isajax) {
+ BSHandoff::handoff($ajaxsocket, "/getbinaries", undef,
"project=$projid", "repository=$repoid", "arch=$arch",
"binaries=$cgi->{'binaries'}");
+ exit(0);
+ }
+ my $r = fetchbinary($pool, $repo, $path, $n, 3);
+ return unless defined($r);
+ }
if ($path =~ /\.rpm$/) {
push @send, {'name' => "$n.rpm", 'filename' => $path};
} else {
@@ -246,9 +289,13 @@ sub getbinaries {
push @send, {'name' => "$n.meta", 'filename' => "$path-MD5SUMS.meta"};
}
}
+ if (defined($serial)) {
+ BSWatcher::serialize_end($serial);
+ forwardevent($cgi, 'scanrepo', $projid, undef, $repoid, $arch);
+ };
undef $repo;
undef $pool;
- BSServer::reply_cpio(\@send);
+ BSWatcher::reply_cpio(\@send);
return undef;
}

@@ -354,24 +401,26 @@ sub getbinarylist_repository {
}

if ($view eq 'cpio') {
-# XXX: DOD
-# if ($cache->{'/url'} && ! -e "$reporoot/$prp/$arch/:full/$c->{'path'}")
{
-# fetchbinary("$reporoot/$prp/$arch/:full/$c->{'path'}", $c,
$cache->{'/url'});
-# }
my @files;
my $pool = BSSolv::pool->new();
my $repo = addrepo_scan($pool, $prp, $arch);
my %names = $repo ? $repo->pkgnames() : ();
my @bins = $cgi->{'binary'} ? @{$cgi->{'binary'}} : sort keys %names;
+ my $serial = BSWatcher::serialize("$reporoot/$prp/$arch");
+ return unless defined($serial);
for my $bin (@bins) {
my $p = $names{$bin};
if (!$p) {
push @files, {'name' => $bin, 'error' => 'not available'};
next;
}
- my $fullpath = $pool->pkg2fullpath($p, $arch);
+ my $fullpath = $reporoot . "/" . $pool->pkg2fullpath($p, $arch);
+ if (!-e $fullpath && $repo->dodurl()) {
+ my $r = fetchbinary($pool, $repo, $fullpath, $bin, 3);
+ return unless defined($r);
+ }
my $fd = gensym;
- if (!open($fd, '<', "$reporoot/$fullpath")) {
+ if (!open($fd, '<', "$fullpath")) {
push @files, {'name' => $bin, 'error' => 'not available'};
next;
}
@@ -379,18 +428,24 @@ sub getbinarylist_repository {
$n .= $1 if $fullpath =~ /(\.rpm|\.deb)$/;
push @files, {'name' => $n, 'filename' => $fd},
}
+ BSWatcher::serialize_end($serial);
+ # XXX: we don't know if a package was fetched
+ forwardevent($cgi, 'scanrepo', $projid, undef, $repoid, $arch);
undef $repo;
undef $pool;
BSWatcher::reply_cpio(\@files);
return undef;
}

+ # FIXME: nearly a duplicate of getbinaryversions()
if ($view eq 'binaryversions') {
my $pool = BSSolv::pool->new();
my $repo = addrepo_scan($pool, $prp, $arch);
my %names = $repo ? $repo->pkgnames() : ();
my @bins = $cgi->{'binary'} ? @{$cgi->{'binary'}} : sort keys %names;
my @res;
+ my $serial = BSWatcher::serialize("$reporoot/$projid/$repoid/$arch") if
$BSStdServer::isajax;
+ return if $BSStdServer::isajax && !defined($serial);
for my $bin (@bins) {
my $p = $names{$bin};
if (!$p) {
@@ -398,14 +453,32 @@ sub getbinarylist_repository {
next;
}
my $path = "$reporoot/".$pool->pkg2fullpath($p, $arch);
+ my $sizek = $pool->pkg2sizek($p);
+ my $hdrmd5 = $pool->pkg2pkgid($p);
+ if (!-e $path && $repo->dodurl()) {
+ if (!$BSStdServer::isajax) {
+ my @args;
+ push @args, "view=$view";
+ push @args, map {"binary=$_"} @{$cgi->{'binary'} || []};
+ BSHandoff::handoff($ajaxsocket,
"/build/$projid/$repoid/$arch/_repository", undef, @args);
+ exit(0);
+ }
+ my $r = fetchbinary($pool, $repo, $path, $bin, 3);
+ return unless defined($r);
+ }
+ if ($hdrmd5 eq 'd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0') {
+ my @s = stat($path);
+ $sizek = ($s[7] + 1023) >> 10;
+ $hdrmd5 = Build::queryhdrmd5($path);
+ }
my $r;
if ($path =~ /\.rpm$/) {
$r = {'name' => "$bin.rpm"};
} else {
$r = {'name' => "$bin.deb"};
}
- $r->{'hdrmd5'} = $pool->pkg2pkgid($p);
- $r->{'sizek'} = $pool->pkg2sizek($p);
+ $r->{'hdrmd5'} = $hdrmd5;
+ $r->{'sizek'} = $sizek;
push @res, $r;
next if $cgi->{'nometa'};
next unless $path =~ s/\.(?:rpm|deb)$//;
@@ -418,6 +491,10 @@ sub getbinarylist_repository {
$r->{'metamd5'} = $ctx->hexdigest();
close F;
}
+ if (defined($serial)) {
+ BSWatcher::serialize_end($serial);
+ forwardevent($cgi, 'scanrepo', $projid, undef, $repoid, $arch);
+ }
undef $repo;
undef $pool;
return ({ 'binary' => \@res }, $BSXML::binaryversionlist);
@@ -442,9 +519,15 @@ sub getbinarylist_repository {
my $r = {'filename' => $view eq 'names' ? $n : $path };
my $id = $pool->pkg2bsid($p);
if ($id) {
- my @s = split('/', $id, 3);
- $r->{'mtime'} = $s[0];
- $r->{'size'} = $s[1];
+ if ($id eq 'dod') {
+ $r->{'mtime'} = '';
+ $r->{'size'} = '';
+ $r->{'filename'} =~ s/.*\///;
+ } else {
+ my @s = split('/', $id, 3);
+ $r->{'mtime'} = $s[0];
+ $r->{'size'} = $s[1];
+ }
} else {
my @s = stat("$reporoot/$prp/$arch/:full/$path");
if (@s) {
@@ -638,33 +721,6 @@ sub getlogfile {
return undef;
}

-#
-# XXX: DOD
-#
-#sub fetchbinary {
-# my ($dest, $c, $url) = @_;
-# return undef unless $c;
-# return undef unless $c->{'rpath'};
-# my $uri = "$url/$c->{'rpath'}";
-# print "fetching: $uri\n";
-# # FIXME: this should be "more" atomic
-# eval {
-# BSRPC::rpc({
-# uri => $uri,
-# maxredirects => 3,
-# filename => "$dest$$",
-# receiver => \&BSHTTP::file_receiver
-# }, undef);
-# };
-# if ($@) {
-# print "failed: $@";
-# unlink("$dest$$");
-# return undef;
-# }
-# rename("$dest$$", $dest) || die("rename $dest$$ $dest: $!\n");
-# return $dest;
-#}
-
sub getbinary_info {
my ($cgi, $projid, $repoid, $arch, $path) = @_;
my @s = stat($path);
@@ -734,6 +790,8 @@ sub getbinary_repository {
}
my $view = $cgi->{'view'} || '';
my $path = "$reporoot/$projid/$repoid/$arch/:full/$bin";
+ my $serial = BSWatcher::serialize("$reporoot/$projid/$repoid/$arch") if
$BSStdServer::isajax;
+ return if $BSStdServer::isajax && !defined($serial);
if (! -f $path) {
# return by name
my $pool = BSSolv::pool->new();
@@ -742,16 +800,28 @@ sub getbinary_repository {
my $p = $rnames{$bin};
die("no such binary '$bin'\n") unless $p;
$path = "$reporoot/".$pool->pkg2fullpath($p, $arch);
+ if (!-e $path && $repo->dodurl()) {
+ if (!$BSStdServer::isajax) {
+ BSHandoff::handoff($ajaxsocket,
"/build/$projid/$repoid/$arch/_repository/$bin", $view ? "view=$view" : undef);
+ exit(0);
+ }
+ my $r = fetchbinary($pool, $repo, $path, $bin, 3);
+ return unless defined($r);
+ }
undef $repo;
undef $pool;
die("$bin: $!\n") unless -f $path;
}
+ if (defined($serial)) {
+ BSWatcher::serialize_end($serial);
+ forwardevent($cgi, 'scanrepo', $projid, undef, $repoid, $arch);
+ }
return getbinary_info($cgi, $projid, $repoid, $arch, $path) if $view eq
'fileinfo' || $view eq 'fileinfo_ext';
die("unknown view '$view'\n") if $view;
my $type = 'application/octet-stream';
$type = 'application/x-rpm' if $path=~ /\.rpm$/;
$type = 'application/x-debian-package' if $path=~ /\.deb$/;
- BSServer::reply_file($path, "Content-Type: $type");
+ BSWatcher::reply_file($path, "Content-Type: $type");
return undef;
}

@@ -2538,7 +2608,10 @@ my $dispatches_ajax = [
'/ajaxstatus' => \&getajaxstatus,
'/build/$project/$repository/$arch/$package/_log nostream:bool?
start:intnum? end:num? view:?' => \&getlogfile,
'/build/$project/$repository/$arch/$package:package_repository view:?
binary:filename*' => \&getbinarylist,
+ '/build/$project/$repository/$arch/$package:package_repository/$filename
view:?' => \&getbinary,
'/_result $prpa+ oldstate:md5? package* code:* withbinarylist:bool?' =>
\&getresult,
+ '/getbinaries $project $repository $arch binaries: nometa:bool?' =>
\&getbinaries,
+ '/getbinaryversions $project $repository $arch binaries: nometa:bool?' =>
\&getbinaryversions,
];

my $conf = {
diff --git a/src/backend/bs_sched b/src/backend/bs_sched
index fb0cc7f..ea96466 100755
--- a/src/backend/bs_sched
+++ b/src/backend/bs_sched
@@ -4344,30 +4344,6 @@ while(1) {
$packstatus{$packid} = 'done';
goto relsynccheck;
}
- if ($BSConfig::enable_download_on_demand && @meta == @new_meta) {
- # unfetched dod packages have dodo... as hdrid
- my @dodbins = grep {/^d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0/} @new_meta;
- if (@dodbins) {
- # package was built with other data than we had, refresh repos and
redo check
- my $rerun;
- s/^.* // for @dodbins;
- for my $bin (@dodbins) {
- my $p = $dep2pkg{$bin};
- next unless $p;
- my $dodprp = $pool->pkg2reponame($p);
- next unless $dodprp;
- delete $repodatas{$dodprp};
- $rerun = 1;
- }
- if ($rerun) {
- unshift @lookat_high, $prp unless @lookat_high && $lookat_high[0]
eq $prp;
- $nharder++;
- $lastcheck{"$prp/$packid"} = $mylastcheck;
- $packstatus{$packid} = 'done';
- goto relsynccheck;
- }
- }
- }
my @diff = diffsortedmd5(0, \@meta, \@new_meta);
print " - $packid ($packtype)\n";
print " $_\n" for @diff;
< Previous Next >
Follow Ups