Mailinglist Archive: opensuse-buildservice (260 mails)

< Previous Next >
Re: [opensuse-buildservice] first draft of a dod implementation
On 2009-12-02 21:49:32 +0100, Marcus Hüwe wrote:
<SNIP>
I'll post the new patch in the next few days.

Here's a new version of the patch (still misses some bits in bs_repserver):
* incorporated all changes (I hope I didn't forget anything)
* rewrote the "fetchbinary" mechanism in bs_repserver:
* only serve one request for a given project+repo+arch at a time and serve
the other requests for the same project+repo+arch afterwards (this way we
can guarantee that the repserver doesn't download the same binaries
multiple times)
* added support for redirects (status 302) to BSWatcher.pm
(if this idea or the implementation is nonsense please tell me and we can
switch back to the old method:) ).

Feedback, objections etc. are welcome.


Marcus
diff --git a/src/backend/BSSolv.xs b/src/backend/BSSolv.xs
index c9af074..722e6c8 100644
--- a/src/backend/BSSolv.xs
+++ b/src/backend/BSSolv.xs
@@ -44,6 +44,7 @@ typedef Expander *BSSolv__expander;
static Id buildservice_id;
static Id buildservice_repocookie;
static Id buildservice_external;
+static Id buildservice_dodurl;

/* make sure bit n is usable */
#define MAPEXP(m, n) ((m)->size < (((n) + 8) >> 3) ? map_grow(m, n + 256) : 0)
@@ -578,8 +579,12 @@ create_considered(Pool *pool, Repo *repoonly, Map
*considered)
continue;
}
}
- else
- continue;
+ else {
+ const char *bsid = solvable_lookup_str(sb, buildservice_id);
+ /* solvables are identical - don't consider dod pkgs */
+ if (bsid && strcmp(bsid, "dod"))
+ continue;
+ }
MAPCLR(considered, pb);
}
best[s->name] = p;
@@ -1112,6 +1117,7 @@ new(packname = "BSSolv::pool")
buildservice_id = str2id(pool, "buildservice:id", 1);
buildservice_repocookie= str2id(pool, "buildservice:repocookie", 1);
buildservice_external = str2id(pool, "buildservice:external", 1);
+ buildservice_dodurl = str2id(pool, "buildservice:dodurl", 1);
pool_freeidhashes(pool);
RETVAL = pool;
}
@@ -1297,6 +1303,9 @@ repofromdata(BSSolv::pool pool, char *name, HV *rhv)
repodata_set_str(data, SOLVID_META, buildservice_repocookie,
REPOCOOKIE);
if (name && !strcmp(name, "/external/"))
repodata_set_void(data, SOLVID_META, buildservice_external);
+ str = hvlookupstr(rhv, "/url", 4);
+ if (str)
+ repodata_set_str(data, SOLVID_META, buildservice_dodurl, str);
repo_internalize(repo);
RETVAL = repo;
}
@@ -1415,7 +1424,11 @@ pkg2fullpath(BSSolv::pool pool, int p, char *myarch)
CODE:
{
unsigned int medianr;
- const char *s = solvable_get_location(pool->solvables + p,
&medianr);
+ char *s = solvable_get_location(pool->solvables + p, &medianr);
+ /* dod: ignore relative url path */
+ char *tmp = strrchr(s, '/');
+ if (tmp)
+ strcpy(s, tmp + 1);
Repo *repo = pool->solvables[p].repo;
s = pool_tmpjoin(pool, myarch, "/:full/", s);
RETVAL = pool_tmpjoin(pool, repo->name, "/", s);
@@ -1624,7 +1637,11 @@ updatefrombins(BSSolv::repo repo, char *dir, ...)
FOR_REPO_SOLVABLES(repo, p, s)
{
const char *str = solvable_lookup_str(s, buildservice_id);
- if (!str)
+ if (str && !strcmp(str, "dod")) {
+ /* keep all packages which have no stat() id (dod pkgs) */
+ MAPSET(&reused, p - repo->start);
+ continue;
+ } else if (!str)
continue;
h = strhash(str) & hm;
hh = HASHCHAIN_START;
@@ -1754,6 +1771,12 @@ isexternal(BSSolv::repo repo)
OUTPUT:
RETVAL

+const char *
+dodurl(BSSolv::repo repo)
+ CODE:
+ RETVAL = repo_lookup_str(repo, SOLVID_META, buildservice_dodurl);
+ OUTPUT:
+ RETVAL



diff --git a/src/backend/BSWatcher.pm b/src/backend/BSWatcher.pm
index 445a133..eeea0f9 100644
--- a/src/backend/BSWatcher.pm
+++ b/src/backend/BSWatcher.pm
@@ -138,6 +138,7 @@ sub deljob {

sub rpc_error {
my ($ev, $err) = @_;
+# print Dumper($ev);
$ev->{'rpcstate'} = 'error';
print "rpc_error: $err\n";
my $uri = $ev->{'rpcuri'};
@@ -153,6 +154,24 @@ sub rpc_error {
}
}

+sub rpc_redirect {
+ my ($ev, $newuri) = @_;
+ print "redirecting to: $newuri\n";
+ $ev->{'rpcstate'} = 'redirecting';
+ my $olduri = $ev->{'rpcuri'};
+ delete $rpcs{$olduri};
+ close $ev->{'fd'} if $ev->{'fd'};
+ delete $ev->{'fd'};
+ for my $jev (@{$ev->{'joblist'} || []}) {
+ $jev->{'rpcdone'} = $olduri;
+ $jev->{'redirecturi'} = $newuri;
+ redo_request($jev);
+ delete $jev->{'rpcdone'};
+ # needed to determine if redirect finished (see sub rpc())
+ #delete $jev->{'redirecturi'};
+ }
+}
+
sub rpc_result {
my ($ev, $res) = @_;
$ev->{'rpcstate'} = 'done';
@@ -672,12 +691,16 @@ sub rpc_recv_handler {
$ans =~ /^(.*?)\n\r?\n(.*)$/s;
my $headers = $1;
$ans = $2;
- if ($status !~ /^200[^\d]/) {
+ my %headers;
+ BSHTTP::gethead(\%headers, $headers);
+ if ($status =~ /^302[^\d]/) {
+ rpc_error($ev, "remote error: 302 redirect found but no location is
specified") unless $headers{'location'};
+ rpc_redirect($ev, $headers{'location'}) if $headers{'location'};
+ return;
+ } elsif ($status !~ /^200[^\d]/) {
rpc_error($ev, "remote error: $status");
return;
}
- my %headers;
- BSHTTP::gethead(\%headers, $headers);
my $param = $ev->{'param'};
if ($headers{'set-cookie'} && $param->{'uri'}) {
my @cookie = split(',', $headers{'set-cookie'});
@@ -793,6 +816,23 @@ sub rpc_connect_handler {
BSEvents::add($ev, 0);
}

+sub makeuri {
+ my ($uri, $isverbatim, @args) = @_;
+ $uri = BSRPC::urlencode($uri) unless $isverbatim;
+ if (@args) {
+ for (@args) {
+ s/([\000-\040<>\"#&\+=%[\177-\377])/sprintf("%%%02X",ord($1))/sge;
+ s/%3D/=/;
+ }
+ if ($uri =~ /\?/) {
+ $uri .= '&'.join('&', @args);
+ } else {
+ $uri .= '?'.join('&', @args);
+ }
+ }
+ return $uri;
+}
+
my $tcpproto = getprotobyname('tcp');

sub rpc {
@@ -807,24 +847,22 @@ sub rpc {
$uri = $param->{'uri'};
@xhdrs = @{$param->{'headers'} || []};
}
- $uri = BSRPC::urlencode($uri) unless $param->{'verbatim_uri'};
- if (@args) {
- for (@args) {
- s/([\000-\040<>\"#&\+=%[\177-\377])/sprintf("%%%02X",ord($1))/sge;
- s/%3D/=/;
- }
- if ($uri =~ /\?/) {
- $uri .= '&'.join('&', @args);
- } else {
- $uri .= '?'.join('&', @args);
- }
- }

- my $rpcuri = $uri;
+ my $rpcuri = makeuri($uri, $param->{'verbatim_uri'}, @args);
$rpcuri .= ";$jev->{'id'}" unless $param->{'joinable'};

- if ($jev->{'rpcdone'} && $rpcuri eq $jev->{'rpcdone'}) {
+ if ($jev->{'rpcdone'} && $jev->{'redirecturi'} && $jev->{'rpcdone'} ne
$jev->{'redirecturi'}) {
+ die("maxredirects exceeded\n") unless $jev->{'maxredirects'};
+ $uri = $param->{'uri'} = $jev->{'redirecturi'};
+ $rpcuri = makeuri($jev->{'redirecturi'}, undef, @args);
+ $rpcuri .= ";$jev->{'id'}" unless $param->{'joinable'};
+ $jev->{'redirecturi'} = $rpcuri;
+ $jev->{'maxredirects'}--;
+ delete $jev->{'rpcdone'};
+ } elsif ($jev->{'rpcdone'} && $rpcuri eq $jev->{'rpcdone'} ||
$jev->{'redirecturi'}) {
die("$jev->{'rpcerror'}\n") if exists $jev->{'rpcerror'};
+ delete $jev->{'redirecturi'};
+ delete $jev->{'maxredirects'};
my $ans = $jev->{'rpcresult'};
if ($xmlargs) {
die("answer is not xml\n") if $ans !~ /<.*?>/s;
@@ -832,7 +870,7 @@ sub rpc {
}
return $ans;
}
-
+ $jev->{'maxredirects'} = $param->{'maxredirects'} if exists
$param->{'maxredirects'} && !exists $jev->{'maxredirects'};
$jev->{'closehandler'} = \&deljob;
if ($rpcs{$rpcuri}) {
my $ev = $rpcs{$rpcuri};
diff --git a/src/backend/bs_repserver b/src/backend/bs_repserver
index 7419d5f..4c583f4 100755
--- a/src/backend/bs_repserver
+++ b/src/backend/bs_repserver
@@ -223,13 +223,46 @@ sub getbinaries {
my $repo = addrepo_scan($pool, $prp, $arch);
my %rnames = $repo ? $repo->pkgnames() : ();
my @send;
+ my $dirty = 0;
+ my $serialmd5 = Digest::MD5::md5_hex("$prp/$arch");
+ my $serial;
+ $serial = BSWatcher::serialize("$reporoot/$serialmd5.lock") if
$BSStdServer::isajax;
+ return undef if $BSStdServer::isajax && !$serial;
+ for my $n (@qbins) {
+ my $p = $rnames{$n};
+ next unless $p;
+ my $path = "$reporoot/" . $pool->pkg2fullpath($p, $arch);
+ next if -e $path;
+ next unless $repo->dodurl();
+ if (!$BSStdServer::isajax) {
+ my @args;
+ push @args, "project=$projid";
+ push @args, "repository=$repoid";
+ push @args, "arch=$arch";
+ push @args, "binaries=$cgi->{'binaries'}";
+ BSHandoff::handoff($ajaxsocket, '/getbinaries', undef, @args);
+ exit(0);
+ }
+ my $r = fetchbinary($pool, $p, $arch, $repo->dodurl());
+ return if $BSStdServer::isajax && !defined($r);
+ $dirty = 1;
+ }
+ BSWatcher::serialize_end($serial) if $BSStdServer::isajax;
+ if ($dirty) {
+ my $ev = { type => 'scanrepo', 'project' => $projid, 'repository' =>
$repoid };
+ my $evname = "scanrepo:${projid}::$repoid";
+ if (-d "$eventdir/$arch") {
+ writexml("$eventdir/$arch/.$evname", "$eventdir/$arch/$evname", $ev,
$BSXML::event);
+ ping($arch);
+ }
+ }
for my $n (@qbins) {
my $p = $rnames{$n};
if (!$p) {
push @send, {'name' => $n, 'error' => 'not available'};
next;
}
- my $path = "$reporoot/".$pool->pkg2fullpath($p, $arch);
+ my $path = "$reporoot/" . $pool->pkg2fullpath($p, $arch);
if ($path =~ /\.rpm$/) {
push @send, {'name' => "$n.rpm", 'filename' => $path};
} else {
@@ -245,7 +278,7 @@ sub getbinaries {
}
undef $repo;
undef $pool;
- BSServer::reply_cpio(\@send);
+ BSWatcher::reply_cpio(\@send);
return undef;
}

@@ -619,29 +652,28 @@ sub getlogfile {
#
# XXX: DOD
#
-#sub fetchbinary {
-# my ($dest, $c, $url) = @_;
-# return undef unless $c;
-# return undef unless $c->{'rpath'};
-# my $uri = "$url/$c->{'rpath'}";
-# print "fetching: $uri\n";
-# # FIXME: this should be "more" atomic
-# eval {
-# BSRPC::rpc({
-# uri => $uri,
-# maxredirects => 3,
-# filename => "$dest$$",
-# receiver => \&BSHTTP::file_receiver
-# }, undef);
-# };
-# if ($@) {
-# print "failed: $@";
-# unlink("$dest$$");
-# return undef;
-# }
-# rename("$dest$$", $dest) || die("rename $dest$$ $dest: $!\n");
-# return $dest;
-#}
+sub fetchbinary {
+ my ($pool, $p, $arch, $url) = @_;
+ my $uri = "$url/" . $pool->pkg2path($p);
+ my $dest = "$reporoot/" . $pool->pkg2fullpath($p, $arch);
+ my $tmp = "$dest$$";
+ my $r;
+ eval {
+ $r = BSWatcher::rpc({
+ 'uri' => $uri,
+ 'maxredirects' => 1,
+ 'filename' => $tmp,
+ 'receiver' => \&BSHTTP::file_receiver,
+ }, undef);
+ };
+ if ($@) {
+ unlink($tmp);
+ die("fetching failed: $@\n");
+ }
+ return if $BSStdServer::isajax && !defined($r);
+ rename($tmp, $dest) || die("rename $tmp $dest: $!\n");
+ return 1;
+}

sub getbinary_repository {
my ($cgi, $projid, $repoid, $arch, $bin) = @_;
@@ -2379,6 +2411,7 @@ my $dispatches_ajax = [
'/build/$project/$repository/$arch/$package/_log nostream:bool?
start:intnum? end:num? view:?' => \&getlogfile,
'/build/$project/$repository/$arch/$package:package_repository view:?
binary:filename*' => \&getbinarylist,
'/_result $prpa+ oldstate:md5? package* code:* withbinarylist:bool?' =>
\&getresult,
+ '/getbinaries $project $repository $arch binaries: nometa:bool?' =>
\&getbinaries,
];

my $conf = {
diff --git a/src/backend/bs_sched b/src/backend/bs_sched
index bb662c5..5ab5f15 100755
--- a/src/backend/bs_sched
+++ b/src/backend/bs_sched
@@ -260,6 +260,22 @@ sub addrepo_scan {
$repodatas{$prp}->{'lastscan'} = time();
return $cache;
}
+ } else {
+ my ($projid) = split('/', $prp, 2);
+ if ($projpacks->{$projid}->{'download'}) {
+ my $doddata = $projpacks->{$projid}->{'download'}->[0];
+ eval {$cache = Meta::parse("$dir/$doddata->{'metafile'}",
"$doddata->{'mtype'}", { 'arch' => [ $myarch ] })};
+ if ($@) {
+ warn("cannot read metadata: $@");
+ return undef;
+ }
+ for (values %$cache) {
+ $_->{'id'} = 'dod';
+ $_->{'hdrmd5'} = 'd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0';
+ }
+ $cache->{'/url'} = $doddata->{'baseurl'};
+ $cache = $pool->repofromdata($prp, $cache);
+ }
}
my @bins;
local *D;
@@ -293,7 +309,7 @@ sub addrepo_scan {
$cache = $pool->repofrombins($prp, $dir, @bins);
$dirty = 1;
}
- if ($dirty && $cache && !$repodatas{$prp}->{'dontwrite'}) {
+ if (($dirty || !-e "$dir.solv") && $cache &&
!$repodatas{$prp}->{'dontwrite'}) {
$cache->tofile("$dir.solv.new");
rename("$dir.solv.new", "$dir.solv") || die("rename $dir.solv.new
$dir.solv: $!\n");
}
@@ -4176,30 +4192,30 @@ while(1) {
$packstatus{$packid} = 'done';
goto relsynccheck;
}
- if (($BSConfig::enable_download_on_demand || $BSConfig::enable_dod) &&
@meta == @new_meta) {
- # unfetched dod packages have dodo... as hdrid
- my @dodbins = grep {/^dodododododododododododododododo/} @new_meta;
- if (@dodbins) {
- # package was built with other data than we had, refresh repos and
redo check
- my $rerun;
- s/^.* // for @dodbins;
- for my $bin (@dodbins) {
- my $p = $dep2pkg{$bin};
- next unless $p;
- my $dodprp = $pool->pkg2reponame($p);
- next unless $dodprp;
- delete $repodatas{$dodprp};
- $rerun = 1;
- }
- if ($rerun) {
- unshift @lookat_high, $prp unless @lookat_high && $lookat_high[0]
eq $prp;
- $nharder++;
- $lastcheck{"$prp/$packid"} = $mylastcheck;
- $packstatus{$packid} = 'done';
- goto relsynccheck;
- }
- }
- }
+# if (($BSConfig::enable_download_on_demand || $BSConfig::enable_dod) &&
@meta == @new_meta) {
+# # unfetched dod packages have dodo... as hdrid
+# my @dodbins = grep {/^dodododododododododododododododo/} @new_meta;
+# if (@dodbins) {
+# # package was built with other data than we had, refresh repos and
redo check
+# my $rerun;
+# s/^.* // for @dodbins;
+# for my $bin (@dodbins) {
+# my $p = $dep2pkg{$bin};
+# next unless $p;
+# my $dodprp = $pool->pkg2reponame($p);
+# next unless $dodprp;
+# delete $repodatas{$dodprp};
+# $rerun = 1;
+# }
+# if ($rerun) {
+# unshift @lookat_high, $prp unless @lookat_high && $lookat_high[0]
eq $prp;
+# $nharder++;
+# $lastcheck{"$prp/$packid"} = $mylastcheck;
+# $packstatus{$packid} = 'done';
+# goto relsynccheck;
+# }
+# }
+# }
my @diff = diffsortedmd5(0, \@meta, \@new_meta);
print " - $packid ($packtype)\n";
print " $_\n" for @diff;
< Previous Next >