[opensuse-buildservice] [PATCH 1/2 for build] Support remote repo
You can pull it from git://gitorious.org/~yyang/opensuse/yyang-build.git
From eed0158f0a85a7fc046a8181c821fbf77e72ab5a Mon Sep 17 00:00:00 2001 From: Yi Yang
Date: Wed, 24 Mar 2010 10:11:00 +0800 Subject: [PATCH] Support remote repo
--- build | 8 ++++-- createrepomddeps | 37 ++++++++++++++++++++++++++++++++++ init_buildsystem | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 4 deletions(-) diff --git a/build b/build index 18f9622..5a4a061 100755 --- a/build +++ b/build @@ -9,6 +9,7 @@ test -z "$BUILD_DIR" && BUILD_DIR=/usr/lib/build test -z "$BUILD_ROOT" && BUILD_ROOT=/var/tmp/build-root +test -z "$CACHE_DIR" && CACHE_DIR=/var/cache/build export BUILD_ARCH BUILD_HOST_ARCH BUILD_ROOT BUILD_RPMS BUILD_DIR @@ -130,6 +131,7 @@ Known Parameters: repository NAME. The repo must be refreshed with zypp so package meta data is available locally. With emtpy NAME all enabled repositories are used. + url can specify a remote repo. --rpms path1:path2:... Specify path where to find the RPMs for the build system @@ -889,7 +891,7 @@ if test -n "$LIST_STATE" ; then } for SPECFILE in $BUILD_ROOT/usr/src/packages/SPECS/*.spec ; do : ; done fi - init_buildsystem --list-state "${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS + init_buildsystem --cachedir $CACHE_DIR --list-state "${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS ERR=$? rm -rf $BUILD_ROOT cleanup_and_exit $ERR @@ -1070,7 +1072,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do if test "$DO_INIT" = true ; then # do fist stage of init_buildsystem rm -f $BUILD_ROOT/.build.success - set -- init_buildsystem --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS + set -- init_buildsystem --cachedir $CACHE_DIR --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS echo "$* ..." "$@" || cleanup_and_exit 1 check_exit @@ -1284,7 +1286,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do echo "BUILD_INCARNATION=$INCARNATION" > $BUILD_ROOT/.buildenv CREATE_BUILD_BINARIES= egrep '^#[ ]*needsbinariesforbuild[ ]*$' >/dev/null <$MYSRCDIR/$SPECFILE && CREATE_BUILD_BINARIES=--create-build-binaries - set -- init_buildsystem "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS + set -- init_buildsystem --cachedir $CACHE_DIR "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS echo "$* ..." "$@" || cleanup_and_exit 1 check_exit diff --git a/createrepomddeps b/createrepomddeps index 61c604a..3b41b9f 100755 --- a/createrepomddeps +++ b/createrepomddeps @@ -9,6 +9,11 @@ use XML::Parser; use Data::Dumper; use Getopt::Long; use Build::Rpm; +use Digest::MD5 qw(md5 md5_hex md5_base64); +use File::Path qw(make_path remove_tree); +use File::Basename; +use LWP::UserAgent; +use URI; Getopt::Long::Configure("no_ignore_case"); my @parent = []; @@ -342,6 +347,30 @@ sub printasbuildcachefile(@) } } +sub geturl +{ + my $url = $_[0]; + my $dest = $_[1]; + + my $ua = LWP::UserAgent->new( + agent => "openSUSE build script", + env_proxy => 1, + timeout => 42); + my $uri = URI->new($url); + my $res = $ua->mirror($uri, $dest.'/'.basename($uri->path)); + die "reqesting $uri failed: ".$res->status_line."\n" unless $res->is_success; +} + +sub getmetadata +{ + my $url = $_[0]; + my $dir = $_[1]; + + my $dest = $dir . "repodata"; + make_path($dest); + geturl($url . "repodata/repomd.xml", $dest); +} + ### main GetOptions ( @@ -378,6 +407,11 @@ for my $url (@ARGV) { } else { die "unsupported repo type: $type\n"; } + } elsif ($url =~ /^http:\/\/([^\/]*)\/?/) { + my $repoid = md5_hex($url); + $dir = "/var/cache/build/$repoid/"; + getmetadata($url, $dir); + $baseurl = $url; } else { $dir = $url; $dir .= '/' unless $dir =~ /\/$/; @@ -395,6 +429,9 @@ for my $url (@ARGV) { @cursor = ([undef, $primaryparser]); my $u = $dir . $f->{'location'}; + unless (-e $u) { + geturl($url . "repodata/" . basename($u), $dir . "repodata/") + } $u = 'gzip -cd ' . $u . '|' if ($u =~ /\.gz$/); # XXX my $fh; diff --git a/init_buildsystem b/init_buildsystem index efa9712..48d39b1 100755 --- a/init_buildsystem +++ b/init_buildsystem @@ -76,6 +76,11 @@ while test -n "$1" ; do CLEAN_BUILD="$1" shift ;; + --cachedir) + shift + CACHE_DIR=$1 + shift + ;; *) break ;; @@ -285,6 +290,10 @@ function validate_cache_file { test -z "$SRC" && SRC=`pwd` if [ "${SRC#zypp://}" != "$SRC" ]; then set -- $BUILD_DIR/createrepomddeps "$SRC" + elif [ "${SRC#http://}" != "$SRC" ]; then + repoid=$(echo -n $SRC | md5sum | cut -d" " -f 1) + mkdir -p $CACHE_DIR/$repoid + set -- $BUILD_DIR/createrepomddeps "$SRC" elif [ ! -e "$SRC" ]; then echo "*** $SRC does not exist" >&2 cleanup_and_exit 1 @@ -361,6 +370,45 @@ download_zypp() fi } +getcachedir() +{ + url=$1 + for repo in "${repos[@]}" ; do + if [ "${url:0:${#repo}}" == "$repo" ] ; then + repoid=$(echo -n $repo | md5sum | cut -d" " -f 1) + echo $CACHE_DIR/$repoid + break + fi + done +} + +download_repo() +{ + local url="$1" + local name=$(basename $url) + local cachedir=$(getcachedir $url) + local destfile=$cachedir/$name + if [ ! -e "$destfile" ]; then + echo "downloading $url to $cachedir... "; + $BUILD_DIR/download "$cachedir" "$url" || cleanup_and_exit 1 + rpm -K "$destfile" > $destfile.v || { echo "rpm verify failed" >&2; rm -f "$destfile"; cleanup_and_exit 1; } + if grep "NOT OK" $destfile.v; then + rm -f "$destfile" + cleanup_and_exit 1 + fi + rm -f "$destfile.v" + fi +} + +get_pkg_filename() +{ + local url="$1" + local name=$(basename $url) + local cachedir=$(getcachedir $url) + local destfile=$cachedir/$name + echo $destfile +} + set_build_arch trap fail_exit EXIT @@ -497,13 +545,21 @@ else zypp://*) download_zypp "$url" ;; + http://*) + download_repo "$url" + ;; *) echo "unsupported url: $url" >&2 cleanup_and_exit 1 ;; esac fi - ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + if [ "${SRC:0:4}" == "http" ] ; then + target_file=$(get_pkg_filename "$SRC") + ln -s "$target_file" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + else + ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + fi PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL $PKG" done < $RPMLIST -- 1.6.0.6 -- To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@opensuse.org For additional commands, e-mail: opensuse-buildservice+help@opensuse.org
Just to clarify why this is needed: At the moment, build already supports building local packages without obs access and without osc using a local, on disk rpmmd repository. Without OBS and osc if you want to build a package, you need a full mirror of a distro, or at least it is not flexible enough, since you would need to always have the needed rpms ready for build in a local directory. This patch makes it possible to point build to a remote repo and the build script would download the needed packages, on demand. Is this a duplication of osc build? Maybe, if you have osc and if the distro you want to build for is hosted in obs, but the scenario this fix addresses assumes you have no obs and the distro you are building for, i.e. it might be an old snapshot, is not in obs and the user has no account on obs... Makes sense? Is the patch acceptable? Anas On 2010-03-26, at 3:29 AM, Yi Yang wrote:
You can pull it from git://gitorious.org/~yyang/opensuse/yyang-build.git
From eed0158f0a85a7fc046a8181c821fbf77e72ab5a Mon Sep 17 00:00:00 2001 From: Yi Yang
Date: Wed, 24 Mar 2010 10:11:00 +0800 Subject: [PATCH] Support remote repo --- build | 8 ++++-- createrepomddeps | 37 ++++++++++++++++++++++++++++++++++ init_buildsystem | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 4 deletions(-)
diff --git a/build b/build index 18f9622..5a4a061 100755 --- a/build +++ b/build @@ -9,6 +9,7 @@
test -z "$BUILD_DIR" && BUILD_DIR=/usr/lib/build test -z "$BUILD_ROOT" && BUILD_ROOT=/var/tmp/build-root +test -z "$CACHE_DIR" && CACHE_DIR=/var/cache/build
export BUILD_ARCH BUILD_HOST_ARCH BUILD_ROOT BUILD_RPMS BUILD_DIR
@@ -130,6 +131,7 @@ Known Parameters: repository NAME. The repo must be refreshed with zypp so package meta data is available locally. With emtpy NAME all enabled repositories are used. + url can specify a remote repo.
--rpms path1:path2:... Specify path where to find the RPMs for the build system @@ -889,7 +891,7 @@ if test -n "$LIST_STATE" ; then } for SPECFILE in $BUILD_ROOT/usr/src/packages/SPECS/*.spec ; do : ; done fi - init_buildsystem --list-state "${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS + init_buildsystem --cachedir $CACHE_DIR --list-state "${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS ERR=$? rm -rf $BUILD_ROOT cleanup_and_exit $ERR @@ -1070,7 +1072,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do if test "$DO_INIT" = true ; then # do fist stage of init_buildsystem rm -f $BUILD_ROOT/.build.success - set -- init_buildsystem --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS + set -- init_buildsystem --cachedir $CACHE_DIR --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS echo "$* ..." "$@" || cleanup_and_exit 1 check_exit @@ -1284,7 +1286,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do echo "BUILD_INCARNATION=$INCARNATION" > $BUILD_ROOT/.buildenv CREATE_BUILD_BINARIES= egrep '^#[ ]*needsbinariesforbuild[ ]*$' >/dev/null <$MYSRCDIR/$SPECFILE && CREATE_BUILD_BINARIES=--create-build-binaries - set -- init_buildsystem "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS + set -- init_buildsystem --cachedir $CACHE_DIR "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS echo "$* ..." "$@" || cleanup_and_exit 1 check_exit diff --git a/createrepomddeps b/createrepomddeps index 61c604a..3b41b9f 100755 --- a/createrepomddeps +++ b/createrepomddeps @@ -9,6 +9,11 @@ use XML::Parser; use Data::Dumper; use Getopt::Long; use Build::Rpm; +use Digest::MD5 qw(md5 md5_hex md5_base64); +use File::Path qw(make_path remove_tree); +use File::Basename; +use LWP::UserAgent; +use URI; Getopt::Long::Configure("no_ignore_case");
my @parent = []; @@ -342,6 +347,30 @@ sub printasbuildcachefile(@) } }
+sub geturl +{ + my $url = $_[0]; + my $dest = $_[1]; + + my $ua = LWP::UserAgent->new( + agent => "openSUSE build script", + env_proxy => 1, + timeout => 42); + my $uri = URI->new($url); + my $res = $ua->mirror($uri, $dest.'/'.basename($uri->path)); + die "reqesting $uri failed: ".$res->status_line."\n" unless $res->is_success; +} + +sub getmetadata +{ + my $url = $_[0]; + my $dir = $_[1]; + + my $dest = $dir . "repodata"; + make_path($dest); + geturl($url . "repodata/repomd.xml", $dest); +} + ### main
GetOptions ( @@ -378,6 +407,11 @@ for my $url (@ARGV) { } else { die "unsupported repo type: $type\n"; } + } elsif ($url =~ /^http:\/\/([^\/]*)\/?/) { + my $repoid = md5_hex($url); + $dir = "/var/cache/build/$repoid/"; + getmetadata($url, $dir); + $baseurl = $url; } else { $dir = $url; $dir .= '/' unless $dir =~ /\/$/; @@ -395,6 +429,9 @@ for my $url (@ARGV) { @cursor = ([undef, $primaryparser]);
my $u = $dir . $f->{'location'}; + unless (-e $u) { + geturl($url . "repodata/" . basename($u), $dir . "repodata/") + } $u = 'gzip -cd ' . $u . '|' if ($u =~ /\.gz$/); # XXX
my $fh; diff --git a/init_buildsystem b/init_buildsystem index efa9712..48d39b1 100755 --- a/init_buildsystem +++ b/init_buildsystem @@ -76,6 +76,11 @@ while test -n "$1" ; do CLEAN_BUILD="$1" shift ;; + --cachedir) + shift + CACHE_DIR=$1 + shift + ;; *) break ;; @@ -285,6 +290,10 @@ function validate_cache_file { test -z "$SRC" && SRC=`pwd` if [ "${SRC#zypp://}" != "$SRC" ]; then set -- $BUILD_DIR/createrepomddeps "$SRC" + elif [ "${SRC#http://}" != "$SRC" ]; then + repoid=$(echo -n $SRC | md5sum | cut -d" " -f 1) + mkdir -p $CACHE_DIR/$repoid + set -- $BUILD_DIR/createrepomddeps "$SRC" elif [ ! -e "$SRC" ]; then echo "*** $SRC does not exist" >&2 cleanup_and_exit 1 @@ -361,6 +370,45 @@ download_zypp() fi }
+getcachedir() +{ + url=$1 + for repo in "${repos[@]}" ; do + if [ "${url:0:${#repo}}" == "$repo" ] ; then + repoid=$(echo -n $repo | md5sum | cut -d" " -f 1) + echo $CACHE_DIR/$repoid + break + fi + done +} + +download_repo() +{ + local url="$1" + local name=$(basename $url) + local cachedir=$(getcachedir $url) + local destfile=$cachedir/$name + if [ ! -e "$destfile" ]; then + echo "downloading $url to $cachedir... "; + $BUILD_DIR/download "$cachedir" "$url" || cleanup_and_exit 1 + rpm -K "$destfile" > $destfile.v || { echo "rpm verify failed" >&2; rm -f "$destfile"; cleanup_and_exit 1; } + if grep "NOT OK" $destfile.v; then + rm -f "$destfile" + cleanup_and_exit 1 + fi + rm -f "$destfile.v" + fi +} + +get_pkg_filename() +{ + local url="$1" + local name=$(basename $url) + local cachedir=$(getcachedir $url) + local destfile=$cachedir/$name + echo $destfile +} + set_build_arch
trap fail_exit EXIT @@ -497,13 +545,21 @@ else zypp://*) download_zypp "$url" ;; + http://*) + download_repo "$url" + ;; *) echo "unsupported url: $url" >&2 cleanup_and_exit 1 ;; esac fi - ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + if [ "${SRC:0:4}" == "http" ] ; then + target_file=$(get_pkg_filename "$SRC") + ln -s "$target_file" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + else + ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + fi PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL $PKG" done < $RPMLIST
-- 1.6.0.6
-- To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@opensuse.org For additional commands, e-mail: opensuse-buildservice+help@opensuse.org
-- To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@opensuse.org For additional commands, e-mail: opensuse-buildservice+help@opensuse.org
Anas Nashif wrote:
Just to clarify why this is needed: At the moment, build already supports building local packages without obs access and without osc using a local, on disk rpmmd repository. Without OBS and osc if you want to build a package, you need a full mirror of a distro, or at least it is not flexible enough, since you would need to always have the needed rpms ready for build in a local directory.
Not anymore. Build can use zypp repos :-) Only the metadata which is cached by zypp anyways needs to be local.
This patch makes it possible to point build to a remote repo and the build script would download the needed packages, on demand. Is this a duplication of osc build? Maybe, if you have osc and if the distro you want to build for is hosted in obs, but the scenario this fix addresses assumes you have no obs and the distro you are building for, i.e. it might be an old snapshot, is not in obs and the user has no account on obs...
Makes sense? Is the patch acceptable?
Sure. It lacks a way to refresh the metadata though. Also, gpg metadata checks are probably a good idea if build downloads stuff itself. That's where it gets more complicated though... cu Ludwig -- (o_ Ludwig Nussel //\ V_/_ http://www.suse.de/ SUSE LINUX Products GmbH, GF: Markus Rex, HRB 16746 (AG Nuernberg) -- To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@opensuse.org For additional commands, e-mail: opensuse-buildservice+help@opensuse.org
On 2010-03-26, at 11:05 AM, Ludwig Nussel wrote:
Anas Nashif wrote:
Just to clarify why this is needed: At the moment, build already supports building local packages without obs access and without osc using a local, on disk rpmmd repository. Without OBS and osc if you want to build a package, you need a full mirror of a distro, or at least it is not flexible enough, since you would need to always have the needed rpms ready for build in a local directory.
Not anymore. Build can use zypp repos :-) Only the metadata which is cached by zypp anyways needs to be local.
Yeah, but not everyone has zypp, Yet...
This patch makes it possible to point build to a remote repo and the build script would download the needed packages, on demand. Is this a duplication of osc build? Maybe, if you have osc and if the distro you want to build for is hosted in obs, but the scenario this fix addresses assumes you have no obs and the distro you are building for, i.e. it might be an old snapshot, is not in obs and the user has no account on obs...
Makes sense? Is the patch acceptable?
Sure. It lacks a way to refresh the metadata though. Also, gpg metadata checks are probably a good idea if build downloads stuff itself. That's where it gets more complicated though...
Ok, will look into that.. Thanks, Anas
cu Ludwig
-- (o_ Ludwig Nussel //\ V_/_ http://www.suse.de/ SUSE LINUX Products GmbH, GF: Markus Rex, HRB 16746 (AG Nuernberg) -- To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@opensuse.org For additional commands, e-mail: opensuse-buildservice+help@opensuse.org
-- To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@opensuse.org For additional commands, e-mail: opensuse-buildservice+help@opensuse.org
On 2010-03-26, at 11:05 AM, Ludwig Nussel wrote:
Anas Nashif wrote:
Just to clarify why this is needed: At the moment, build already supports building local packages without obs access and without osc using a local, on disk rpmmd repository. Without OBS and osc if you want to build a package, you need a full mirror of a distro, or at least it is not flexible enough, since you would need to always have the needed rpms ready for build in a local directory.
Not anymore. Build can use zypp repos :-) Only the metadata which is cached by zypp anyways needs to be local.
Yeah, but not everyone has zypp, Yet...
If a user is using Fedora, he/she can't get zypper, this patch can meet his/her requirements on Fedora. :-)
This patch makes it possible to point build to a remote repo and the build script would download the needed packages, on demand. Is this a
duplication
of osc build? Maybe, if you have osc and if the distro you want to build for is hosted in obs, but the scenario this fix addresses assumes you have no obs and the distro you are building for, i.e. it might be an old snapshot, is not in obs and the user has no account on obs...
Makes sense? Is the patch acceptable?
Sure. It lacks a way to refresh the metadata though. Also, gpg metadata checks are probably a good idea if build downloads stuff itself. That's where it gets more complicated though...
Metadata will be refreshed when you run build, yes, to check gpg is necessary. -- To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@opensuse.org For additional commands, e-mail: opensuse-buildservice+help@opensuse.org
participants (4)
-
Anas Nashif
-
Ludwig Nussel
-
Yang, Yi Y
-
Yi Yang