Re: [opensuse-buildservice] [PATCH 1/2 for build] Support remote repo

Just to clarify why this is needed: At the moment, build already supports building local packages without obs access and without osc using a local, on disk rpmmd repository. Without OBS and osc if you want to build a package, you need a full mirror of a distro, or at least it is not flexible enough, since you would need to always have the needed rpms ready for build in a local directory. This patch makes it possible to point build to a remote repo and the build script would download the needed packages, on demand. Is this a duplication of osc build? Maybe, if you have osc and if the distro you want to build for is hosted in obs, but the scenario this fix addresses assumes you have no obs and the distro you are building for, i.e. it might be an old snapshot, is not in obs and the user has no account on obs... Makes sense? Is the patch acceptable? Anas On 2010-03-26, at 3:29 AM, Yi Yang wrote:

You can pull it from git://gitorious.org/~yyang/opensuse/yyang-build.git

From eed0158f0a85a7fc046a8181c821fbf77e72ab5a Mon Sep 17 00:00:00 2001 From: Yi Yang Date: Wed, 24 Mar 2010 10:11:00 +0800 Subject: [PATCH] Support remote repo

--- build | 8 ++++-- createrepomddeps | 37 ++++++++++++++++++++++++++++++++++ init_buildsystem | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 4 deletions(-)

diff --git a/build b/build index 18f9622..5a4a061 100755 --- a/build +++ b/build @@ -9,6 +9,7 @@

test -z "$BUILD_DIR" && BUILD_DIR=/usr/lib/build test -z "$BUILD_ROOT" && BUILD_ROOT=/var/tmp/build-root +test -z "$CACHE_DIR" && CACHE_DIR=/var/cache/build

export BUILD_ARCH BUILD_HOST_ARCH BUILD_ROOT BUILD_RPMS BUILD_DIR

@@ -130,6 +131,7 @@ Known Parameters: repository NAME. The repo must be refreshed with zypp so package meta data is available locally. With emtpy NAME all enabled repositories are used. + url can specify a remote repo.

--rpms path1:path2:... Specify path where to find the RPMs for the build system @@ -889,7 +891,7 @@ if test -n "$LIST_STATE" ; then } for SPECFILE in $BUILD_ROOT/usr/src/packages/SPECS/*.spec ; do : ; done fi - init_buildsystem --list-state "${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS + init_buildsystem --cachedir $CACHE_DIR --list-state "${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS ERR=$? rm -rf $BUILD_ROOT cleanup_and_exit $ERR @@ -1070,7 +1072,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do if test "$DO_INIT" = true ; then # do fist stage of init_buildsystem rm -f $BUILD_ROOT/.build.success - set -- init_buildsystem --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS + set -- init_buildsystem --cachedir $CACHE_DIR --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS echo "$* ..." "$@" || cleanup_and_exit 1 check_exit @@ -1284,7 +1286,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do echo "BUILD_INCARNATION=$INCARNATION" > $BUILD_ROOT/.buildenv CREATE_BUILD_BINARIES= egrep '^#[ ]*needsbinariesforbuild[ ]*$' >/dev/null <$MYSRCDIR/$SPECFILE && CREATE_BUILD_BINARIES=--create-build-binaries - set -- init_buildsystem "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS + set -- init_buildsystem --cachedir $CACHE_DIR "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS echo "$* ..." "$@" || cleanup_and_exit 1 check_exit diff --git a/createrepomddeps b/createrepomddeps index 61c604a..3b41b9f 100755 --- a/createrepomddeps +++ b/createrepomddeps @@ -9,6 +9,11 @@ use XML::Parser; use Data::Dumper; use Getopt::Long; use Build::Rpm; +use Digest::MD5 qw(md5 md5_hex md5_base64); +use File::Path qw(make_path remove_tree); +use File::Basename; +use LWP::UserAgent; +use URI; Getopt::Long::Configure("no_ignore_case");

my @parent = []; @@ -342,6 +347,30 @@ sub printasbuildcachefile(@) } }

+sub geturl +{ + my $url = $_[0]; + my $dest = $_[1]; + + my $ua = LWP::UserAgent->new( + agent => "openSUSE build script", + env_proxy => 1, + timeout => 42); + my $uri = URI->new($url); + my $res = $ua->mirror($uri, $dest.'/'.basename($uri->path)); + die "reqesting $uri failed: ".$res->status_line."\n" unless $res->is_success; +} + +sub getmetadata +{ + my $url = $_[0]; + my $dir = $_[1]; + + my $dest = $dir . "repodata"; + make_path($dest); + geturl($url . "repodata/repomd.xml", $dest); +} + ### main

GetOptions ( @@ -378,6 +407,11 @@ for my $url (@ARGV) { } else { die "unsupported repo type: $type\n"; } + } elsif ($url =~ /^http:\/\/([^\/]*)\/?/) { + my $repoid = md5_hex($url); + $dir = "/var/cache/build/$repoid/"; + getmetadata($url, $dir); + $baseurl = $url; } else { $dir = $url; $dir .= '/' unless $dir =~ /\/$/; @@ -395,6 +429,9 @@ for my $url (@ARGV) { @cursor = ([undef, $primaryparser]);

my $u = $dir . $f->{'location'}; + unless (-e $u) { + geturl($url . "repodata/" . basename($u), $dir . "repodata/") + } $u = 'gzip -cd ' . $u . '|' if ($u =~ /\.gz$/); # XXX

my $fh; diff --git a/init_buildsystem b/init_buildsystem index efa9712..48d39b1 100755 --- a/init_buildsystem +++ b/init_buildsystem @@ -76,6 +76,11 @@ while test -n "$1" ; do CLEAN_BUILD="$1" shift ;; + --cachedir) + shift + CACHE_DIR=$1 + shift + ;; *) break ;; @@ -285,6 +290,10 @@ function validate_cache_file { test -z "$SRC" && SRC=`pwd` if [ "${SRC#zypp://}" != "$SRC" ]; then set -- $BUILD_DIR/createrepomddeps "$SRC" + elif [ "${SRC#http://}" != "$SRC" ]; then + repoid=$(echo -n $SRC | md5sum | cut -d" " -f 1) + mkdir -p $CACHE_DIR/$repoid + set -- $BUILD_DIR/createrepomddeps "$SRC" elif [ ! -e "$SRC" ]; then echo "*** $SRC does not exist" >&2 cleanup_and_exit 1 @@ -361,6 +370,45 @@ download_zypp() fi }

+getcachedir() +{ + url=$1 + for repo in "${repos[@]}" ; do + if [ "${url:0:${#repo}}" == "$repo" ] ; then + repoid=$(echo -n $repo | md5sum | cut -d" " -f 1) + echo $CACHE_DIR/$repoid + break + fi + done +} + +download_repo() +{ + local url="$1" + local name=$(basename $url) + local cachedir=$(getcachedir $url) + local destfile=$cachedir/$name + if [ ! -e "$destfile" ]; then + echo "downloading $url to $cachedir... "; + $BUILD_DIR/download "$cachedir" "$url" || cleanup_and_exit 1 + rpm -K "$destfile" > $destfile.v || { echo "rpm verify failed" >&2; rm -f "$destfile"; cleanup_and_exit 1; } + if grep "NOT OK" $destfile.v; then + rm -f "$destfile" + cleanup_and_exit 1 + fi + rm -f "$destfile.v" + fi +} + +get_pkg_filename() +{ + local url="$1" + local name=$(basename $url) + local cachedir=$(getcachedir $url) + local destfile=$cachedir/$name + echo $destfile +} + set_build_arch

trap fail_exit EXIT @@ -497,13 +545,21 @@ else zypp://*) download_zypp "$url" ;; + http://*) + download_repo "$url" + ;; *) echo "unsupported url: $url" >&2 cleanup_and_exit 1 ;; esac fi - ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + if [ "${SRC:0:4}" == "http" ] ; then + target_file=$(get_pkg_filename "$SRC") + ln -s "$target_file" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + else + ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + fi PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL $PKG" done < $RPMLIST

-- 1.6.0.6

-- To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@opensuse.org For additional commands, e-mail: opensuse-buildservice+help@opensuse.org

-- To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@opensuse.org For additional commands, e-mail: opensuse-buildservice+help@opensuse.org