Mailinglist Archive: opensuse-buildservice (245 mails)

< Previous Next >
Re: [opensuse-buildservice] How can I configure download on demand to use an RPM repository?
On 2010-09-29 13:57:11 -0500, Michael E Brown wrote:
On Wed, Sep 29, 2010 at 04:42:56AM -0500, Marcus Hüwe wrote:
On 2010-09-28 11:31:38 -0500, Michael E Brown wrote:
On Tue, Sep 28, 2010 at 11:14:12AM -0500, Robert Xu wrote:
On Tue, Sep 28, 2010 at 06:24, Marcus Hüwe <suse-tux@xxxxxx> wrote:

So would this let us get RPMs that are continually updated, like from
a repo such as Fedora Updates?

Absolutely not. In fact, this seriously breaks and wont work at all due
to some
limitations of DoD. The repo you pull from has to be completely static.
The
build service never re-downloads the repo metadata, and caches a lot of
info.
Can you summarize which features do you miss? If I understand you
correctly the obs should fetch the required metadata on its own?

Correct. Today you have to manually download primary.xml.gz and uncompress
even
though the information exists where OBS could/should get this itself. It
should
download repomd.xml and parse that to get the url for primary.xml.gz, then
download/uncompress.

In order to support non static metadata (without "hacks" like
removing :full* and triggering a scanrepo event) we would just
need to compare the old metadata with the new metadata and remove
some entries from :full.solv (and their correspondent binaries from
the :full/ dir). I think this shouldn't be too hard to implement:)

Well, if you put it that way, maybe. But you still need to teach OBS to
download the metadata in the first place.

I just wrote a small patch to support the "update dod meta" feature.
If you set up an initial dod repo the scheduler downloads the
required metadata automatically. Additionally you can create a so
called "refreshdodmeta" event so that the scheduler updates the dod
metadata and cleans up the :full dir (removes unneeded binaries).

TODO:
* api support for the refreshdodmeta event (trivial)
* add download support for susetags and debmd (probably also trivial)
* support project meta changes (for instance update the dodmeta if
the <download /> tag has changed, remove binaries if the
<download /> tag is removed? etc.)

Feedback, objections to the patch/the current design are welcome:)


Marcus
diff --git a/src/backend/Meta.pm b/src/backend/Meta.pm
index de9b6b9..32c9a79 100644
--- a/src/backend/Meta.pm
+++ b/src/backend/Meta.pm
@@ -36,4 +36,9 @@ sub parse {
return Meta::Susetagsmd::parse($fn, $opts) if $type eq 'susetagsmd';
}

+sub download {
+ my ($baseurl, $fn, $type, $opts) = @_;
+ return Meta::Rpmmd::download($baseurl, $fn, $opts) if $type eq 'rpmmd';
+}
+
1;
diff --git a/src/backend/Meta/Rpmmd.pm b/src/backend/Meta/Rpmmd.pm
index e9d116c..0a57d1f 100644
--- a/src/backend/Meta/Rpmmd.pm
+++ b/src/backend/Meta/Rpmmd.pm
@@ -26,6 +26,12 @@ package Meta::Rpmmd;
use strict;
use warnings;
use XML::Parser;
+use BSRPC;
+use BSHTTP;
+use BSConfig;
+# XXX: implement a fallback if Compress::Zlib is not available (see
Build/Deb.pm)
+use Compress::Zlib;
+use Data::Dumper;

sub parse {
my ($fn, $opts) = @_;
@@ -42,6 +48,31 @@ sub parse {
return $h->getrepodata();
}

+sub download {
+ my ($baseurl, $fn, $opts) = @_;
+ my $param = {
+ 'uri' => "$baseurl/repodata/primary.xml.gz",
+ 'filename' => "$fn.$$",
+ 'receiver' => \&BSHTTP::file_receiver,
+ 'maxredirects' => $opts->{'maxredirects'} || 0,
+ };
+ my $r = BSRPC::rpc($param);
+ if ($opts->{'uncompress'}) {
+ local *F;
+ my $gz = gzopen("$fn.$$", 'r') || die("gzopen $fn.$$: $!\n");
+ open(F, ">", $fn) || die("open $fn: $!\n");
+ my $buf;
+ print F $buf while $gz->gzread($buf) > 0;
+ die("gzread: $gzerrno\n") if $gzerrno != Z_STREAM_END;
+ $gz->gzclose();
+ close(F);
+ unlink("$fn.$$");
+ } else {
+ rename("$fn.$$", $fn) || die("rename $fn.$$ $fn: $!\n");
+ }
+ return $r;
+}
+
1;

package rpmmdhandler;
diff --git a/src/backend/bs_repserver b/src/backend/bs_repserver
index e3c1c5f..3e8e1e1 100755
--- a/src/backend/bs_repserver
+++ b/src/backend/bs_repserver
@@ -1557,6 +1557,8 @@ sub docommand {
forwardevent($cgi, 'wipe', $projid, $packid, $repoid, $arch);
}
}
+ } elsif ($cmd eq 'refreshdodmeta') {
+ forwardevent($cgi, 'refreshdodmeta', $projid, undef, $repoid, $arch);
}
}
return $BSStdServer::return_ok;
diff --git a/src/backend/bs_sched b/src/backend/bs_sched
index 11403b5..a3f7b56 100755
--- a/src/backend/bs_sched
+++ b/src/backend/bs_sched
@@ -81,6 +81,7 @@ my $extrepodb = "$BSConfig::bsdir/db/published";
my $uploaddir = "$BSConfig::bsdir/upload";
my $rundir = $BSConfig::rundir || "$BSConfig::bsdir/run";
my $infodir = "$BSConfig::bsdir/info";
+my $dodmetadir = "$BSConfig::bsdir/dodmeta";

if ($ARGV[0] eq '--testmode') {
$testmode = 1;
@@ -261,23 +262,8 @@ sub addrepo_scan {
return $cache;
}
} elsif ($BSConfig::enable_download_on_demand) {
- my ($projid) = split('/', $prp, 2);
- my @doddata = grep {$_->{'arch'} && $_->{'arch'} eq $myarch}
@{$projpacks->{$projid}->{'download'} || []};
- if (@doddata) {
- my $doddata = $doddata[0];
- eval {$cache = Meta::parse("$dir/$doddata->{'metafile'}",
$doddata->{'mtype'}, { 'arch' => [ $myarch ] })};
- if ($@) {
- warn("download on demand: cannot read metadata: $@");
- return undef;
- }
- for (values %$cache) {
- $_->{'id'} = 'dod';
- $_->{'hdrmd5'} = 'd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0';
- }
- $cache->{'/url'} = $doddata->{'baseurl'};
- $cache = $pool->repofromdata($prp, $cache);
- $dirty = 1;
- }
+ $cache = refresh_dodmeta($pool, $prp, 1);
+ return $cache if defined($cache);
}
my @bins;
local *D;
@@ -326,6 +312,95 @@ sub enabled {
}


+sub refresh_dodmeta {
+ my ($pool, $prp, $initial) = @_;
+ return undef unless $BSConfig::enable_download_on_demand;
+ my $projid = (split('/', $prp, 2))[0];
+ my @doddata = grep {$_->{'arch'} && $_->{'arch'} eq $myarch}
@{$projpacks->{$projid}->{'download'} || []};
+ return undef unless @doddata;
+ my $doddata = $doddata[0];
+ mkdir_p("$dodmetadir/$prp/$myarch") unless -d "$dodmetadir/$prp/$myarch";
+ # download new meta
+ my $metafile = "$dodmetadir/$prp/$myarch/$doddata->{'metafile'}";
+ eval {
+ print " fetching metadata for $prp\n";
+ Meta::download($doddata->{'baseurl'}, "$metafile.new",
$doddata->{'mtype'}, {'uncompress' => 1, 'maxredirects' => 3});
+ };
+ if ($@) {
+ print " error fetching metadata: $@\n";
+ unlink("$metafile.new");
+ return undef;
+ }
+ my $orepo;
+ $orepo = addrepo($pool, $prp) unless $initial;
+ my $data;
+ eval {
+ $data = Meta::parse("$metafile.new", $doddata->{'mtype'}, {'arch' =>
[$myarch]});
+ };
+ if ($@) {
+ print " unable to parse new metadata: $@\n";
+ # keep metafile.new for debugging purposes
+ return undef;
+ }
+ for (values %$data) {
+ $_->{'id'} = 'dod';
+ $_->{'hdrmd5'} = 'd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0';
+ }
+ $data->{'/url'} = $doddata->{'baseurl'};
+
+ my %onames;
+ %onames = $orepo->pkgnames() unless $initial;
+ my $repo = $pool->repofromdata($projid, $data);
+ my %names = $repo->pkgnames();
+ my @todel;
+ for my $o (keys %onames) {
+ next if $pool->pkg2bsid($onames{$o}) eq 'dod';
+ if (!$names{$o}) {
+ push @todel, $pool->pkg2fullpath($onames{$o}, $myarch);
+ next;
+ }
+ my $r;
+ my $s1 = $pool->pkg2data($onames{$o});
+ my $s2 = $pool->pkg2data($names{$o});
+ $r = $s1->{'name'} eq $s2->{'name'};
+ $r = ($s1->{'epoch'} || '0') eq ($s2->{'epoch'} || '0') ? $r : 0;
+ $r = $s1->{'version'} eq $s2->{'version'} ? $r : 0;
+ $r = $s1->{'release'} eq $s2->{'release'} ? $r : 0;
+ for my $type qw(provides requires) {
+ my %d1 = map {$_ => 1} @{$s1->{$type} || []};
+ my %d2 = map {$_ => 1} @{$s2->{$type} || []};
+ $r = keys(%d1) == keys(%d2) && (grep {!exists $d1{$_}} keys(%d2)) == 0 ?
$r : 0;
+ }
+ push @todel, $pool->pkg2fullpath($onames{$o}, $myarch) unless $r;
+ }
+ unlink("$reporoot/$_") for @todel;
+ print " removed " . @todel . " binaries\n" if @todel;
+ # collect all remaining binaries and update repo
+ my $dir = "$reporoot/$prp/$myarch/:full";
+ local *D;
+ opendir(D, $dir) || die("opendir $dir: $!\n");
+ my @bins = grep {/\.(?:rpm|deb)$/} readdir(D);
+ closedir(D);
+ if (!@bins && -s "$dir.subdirs") {
+ for my $subdir (split(' ', readstr("$dir.subdirs"))) {
+ push @bins, map {"$subdir/$_"} grep {/\.(?:rpm|deb)$/}
ls("$dir/$subdir");
+ }
+ }
+ for (splice @bins) {
+ my @s = stat("$dir/$_");
+ next unless @s;
+ push @bins, $_, "$s[9]/$s[7]/$s[1]";
+ }
+ $repo->updatefrombins($dir, @bins);
+ if (!$repodatas{$prp}->{'dontwrite'}) {
+ $repo->tofile("$dir.solv.new");
+ rename("$dir.solv.new", "$dir.solv") || die("rename $dir.solv.new
$dir.solv: $!\n");
+ }
+ rename("$metafile.new", $metafile) || die("rename $metafile.new $metafile:
$!\n");
+ $repodatas{$prp}->{'lastscan'} = time();
+ return $repo;
+}
+

# this is basically getconfig from the source server
# we do not need any macros, just the config
@@ -3634,6 +3709,24 @@ while(1) {
next;
}

+ if ($ev->{'type'} eq 'refreshdodmeta') {
+ my $projid = $ev->{'project'};
+ my $repoid = $ev->{'repository'};
+ unless ($projid && $repoid) {
+ print "invalid refreshdodmeta event (ignoring...)\n";
+ next;
+ }
+ my $prp = "$projid/$repoid";
+ print "refreshing dod metadata for $prp\n";
+ delete $repodatas{$prp};
+ my $pool = BSSolv::pool->new();
+ refresh_dodmeta($pool, $prp);
+ undef $pool;
+ $changed_high{$prp} = 2;
+ delete $repounchanged{$prp};
+ next;
+ }
+
if ($ev->{'type'} eq 'dumprepo') {
my $prp = "$ev->{'project'}/$ev->{'repository'}";
my $repodata = $repodatas{$prp} || {};
diff --git a/src/backend/bs_srcserver b/src/backend/bs_srcserver
index 65fea52..9f5ad49 100755
--- a/src/backend/bs_srcserver
+++ b/src/backend/bs_srcserver
@@ -3921,7 +3921,7 @@ sub docommand {
} else {
@packids = findpackages($projid, $proj);
}
- die("no packages defined\n") unless @packids;
+ die("no packages defined\n") unless @packids || $cgi->{'nopackages'};

# XXX FIXME multiple repo handling
my $res;
@@ -5380,7 +5380,7 @@ my $dispatches = [
'/build/_workerstatus scheduleronly:bool? arch*' => \&getworkerstatus,
'PUT:/build/_dispatchprios' => \&putdispatchprios,
'/build/_dispatchprios' => \&getdispatchprios,
- 'POST:/build/$project cmd: repository* arch* package* code:*' => \&docommand,
+ 'POST:/build/$project cmd: repository* arch* package* code:*
nopackages:bool?' => \&docommand,
'/build/$project' => \&getrepositorylist,
'/build/$project/_result oldstate:md5? view:resultview* lastbuild:bool?
repository* arch* package* code:*' => \&getresult,
'/build/$project/$repository' => \&getarchlist,
< Previous Next >
Follow Ups