Mailinglist Archive: opensuse-buildservice (216 mails)

< Previous Next >
Re: [opensuse-buildservice] [PATCH 1/2 for build] Support remote repo
  • From: Anas Nashif <nashif@xxxxxxxxxxxxxxx>
  • Date: Fri, 26 Mar 2010 10:33:38 -0400
  • Message-id: <303FFD63-85C3-4ABF-9769-E22C61A5B741@xxxxxxxxxxxxxxx>
Just to clarify why this is needed:
At the moment, build already supports building local packages without obs
access and without osc using a local, on disk rpmmd repository.
Without OBS and osc if you want to build a package, you need a full mirror of a
distro, or at least it is not flexible enough, since you would need to always
have the needed rpms ready for build in a local directory.

This patch makes it possible to point build to a remote repo and the build
script would download the needed packages, on demand.
Is this a duplication of osc build? Maybe, if you have osc and if the distro
you want to build for is hosted in obs, but the scenario this fix addresses
assumes you have no obs and the distro you are building for, i.e. it might be
an old snapshot, is not in obs and the user has no account on obs...

Makes sense? Is the patch acceptable?

Anas


On 2010-03-26, at 3:29 AM, Yi Yang wrote:

You can pull it from git://gitorious.org/~yyang/opensuse/yyang-build.git

From eed0158f0a85a7fc046a8181c821fbf77e72ab5a Mon Sep 17 00:00:00 2001
From: Yi Yang <yi.y.yang@xxxxxxxxx>
Date: Wed, 24 Mar 2010 10:11:00 +0800
Subject: [PATCH] Support remote repo

---
build | 8 ++++--
createrepomddeps | 37 ++++++++++++++++++++++++++++++++++
init_buildsystem | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 99 insertions(+), 4 deletions(-)

diff --git a/build b/build
index 18f9622..5a4a061 100755
--- a/build
+++ b/build
@@ -9,6 +9,7 @@

test -z "$BUILD_DIR" && BUILD_DIR=/usr/lib/build
test -z "$BUILD_ROOT" && BUILD_ROOT=/var/tmp/build-root
+test -z "$CACHE_DIR" && CACHE_DIR=/var/cache/build

export BUILD_ARCH BUILD_HOST_ARCH BUILD_ROOT BUILD_RPMS BUILD_DIR

@@ -130,6 +131,7 @@ Known Parameters:
repository NAME. The repo must be refreshed with zypp
so package meta data is available locally. With emtpy
NAME all enabled repositories are used.
+ url can specify a remote repo.

--rpms path1:path2:...
Specify path where to find the RPMs for the build system
@@ -889,7 +891,7 @@ if test -n "$LIST_STATE" ; then
}
for SPECFILE in $BUILD_ROOT/usr/src/packages/SPECS/*.spec ; do : ; done
fi
- init_buildsystem --list-state "${definesnstuff[@]}" "${repos[@]}"
$USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS
+ init_buildsystem --cachedir $CACHE_DIR --list-state
"${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE
$BUILD_EXTRA_PACKS
ERR=$?
rm -rf $BUILD_ROOT
cleanup_and_exit $ERR
@@ -1070,7 +1072,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do
if test "$DO_INIT" = true ; then
# do fist stage of init_buildsystem
rm -f $BUILD_ROOT/.build.success
- set -- init_buildsystem --prepare "${definesnstuff[@]}"
"${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE"
$ADDITIONAL_PACKS
+ set -- init_buildsystem --cachedir $CACHE_DIR --prepare
"${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST
"$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS
echo "$* ..."
"$@" || cleanup_and_exit 1
check_exit
@@ -1284,7 +1286,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do
echo "BUILD_INCARNATION=$INCARNATION" > $BUILD_ROOT/.buildenv
CREATE_BUILD_BINARIES=
egrep '^#[ ]*needsbinariesforbuild[ ]*$' >/dev/null
<$MYSRCDIR/$SPECFILE && CREATE_BUILD_BINARIES=--create-build-binaries
- set -- init_buildsystem "${definesnstuff[@]}" "${repos[@]}"
$CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST
"$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS
+ set -- init_buildsystem --cachedir $CACHE_DIR "${definesnstuff[@]}"
"${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST
"$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS
echo "$* ..."
"$@" || cleanup_and_exit 1
check_exit
diff --git a/createrepomddeps b/createrepomddeps
index 61c604a..3b41b9f 100755
--- a/createrepomddeps
+++ b/createrepomddeps
@@ -9,6 +9,11 @@ use XML::Parser;
use Data::Dumper;
use Getopt::Long;
use Build::Rpm;
+use Digest::MD5 qw(md5 md5_hex md5_base64);
+use File::Path qw(make_path remove_tree);
+use File::Basename;
+use LWP::UserAgent;
+use URI;
Getopt::Long::Configure("no_ignore_case");

my @parent = [];
@@ -342,6 +347,30 @@ sub printasbuildcachefile(@)
}
}

+sub geturl
+{
+ my $url = $_[0];
+ my $dest = $_[1];
+
+ my $ua = LWP::UserAgent->new(
+ agent => "openSUSE build script",
+ env_proxy => 1,
+ timeout => 42);
+ my $uri = URI->new($url);
+ my $res = $ua->mirror($uri, $dest.'/'.basename($uri->path));
+ die "reqesting $uri failed: ".$res->status_line."\n" unless
$res->is_success;
+}
+
+sub getmetadata
+{
+ my $url = $_[0];
+ my $dir = $_[1];
+
+ my $dest = $dir . "repodata";
+ make_path($dest);
+ geturl($url . "repodata/repomd.xml", $dest);
+}
+
### main

GetOptions (
@@ -378,6 +407,11 @@ for my $url (@ARGV) {
} else {
die "unsupported repo type: $type\n";
}
+ } elsif ($url =~ /^http:\/\/([^\/]*)\/?/) {
+ my $repoid = md5_hex($url);
+ $dir = "/var/cache/build/$repoid/";
+ getmetadata($url, $dir);
+ $baseurl = $url;
} else {
$dir = $url;
$dir .= '/' unless $dir =~ /\/$/;
@@ -395,6 +429,9 @@ for my $url (@ARGV) {
@cursor = ([undef, $primaryparser]);

my $u = $dir . $f->{'location'};
+ unless (-e $u) {
+ geturl($url . "repodata/" . basename($u), $dir . "repodata/")
+ }
$u = 'gzip -cd ' . $u . '|' if ($u =~ /\.gz$/); # XXX

my $fh;
diff --git a/init_buildsystem b/init_buildsystem
index efa9712..48d39b1 100755
--- a/init_buildsystem
+++ b/init_buildsystem
@@ -76,6 +76,11 @@ while test -n "$1" ; do
CLEAN_BUILD="$1"
shift
;;
+ --cachedir)
+ shift
+ CACHE_DIR=$1
+ shift
+ ;;
*)
break
;;
@@ -285,6 +290,10 @@ function validate_cache_file {
test -z "$SRC" && SRC=`pwd`
if [ "${SRC#zypp://}" != "$SRC" ]; then
set -- $BUILD_DIR/createrepomddeps "$SRC"
+ elif [ "${SRC#http://}"; != "$SRC" ]; then
+ repoid=$(echo -n $SRC | md5sum | cut -d" " -f 1)
+ mkdir -p $CACHE_DIR/$repoid
+ set -- $BUILD_DIR/createrepomddeps "$SRC"
elif [ ! -e "$SRC" ]; then
echo "*** $SRC does not exist" >&2
cleanup_and_exit 1
@@ -361,6 +370,45 @@ download_zypp()
fi
}

+getcachedir()
+{
+ url=$1
+ for repo in "${repos[@]}" ; do
+ if [ "${url:0:${#repo}}" == "$repo" ] ; then
+ repoid=$(echo -n $repo | md5sum | cut -d" " -f 1)
+ echo $CACHE_DIR/$repoid
+ break
+ fi
+ done
+}
+
+download_repo()
+{
+ local url="$1"
+ local name=$(basename $url)
+ local cachedir=$(getcachedir $url)
+ local destfile=$cachedir/$name
+ if [ ! -e "$destfile" ]; then
+ echo "downloading $url to $cachedir... ";
+ $BUILD_DIR/download "$cachedir" "$url" || cleanup_and_exit 1
+ rpm -K "$destfile" > $destfile.v || { echo "rpm verify failed" >&2; rm
-f "$destfile"; cleanup_and_exit 1; }
+ if grep "NOT OK" $destfile.v; then
+ rm -f "$destfile"
+ cleanup_and_exit 1
+ fi
+ rm -f "$destfile.v"
+ fi
+}
+
+get_pkg_filename()
+{
+ local url="$1"
+ local name=$(basename $url)
+ local cachedir=$(getcachedir $url)
+ local destfile=$cachedir/$name
+ echo $destfile
+}
+
set_build_arch

trap fail_exit EXIT
@@ -497,13 +545,21 @@ else
zypp://*)
download_zypp "$url"
;;
+ http://*)
+ download_repo "$url"
+ ;;
*)
echo "unsupported url: $url" >&2
cleanup_and_exit 1
;;
esac
fi
- ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}"
+ if [ "${SRC:0:4}" == "http" ] ; then
+ target_file=$(get_pkg_filename "$SRC")
+ ln -s "$target_file"
"$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}"
+ else
+ ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}"
+ fi
PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL $PKG"
done < $RPMLIST

--
1.6.0.6

--
To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@xxxxxxxxxxxx
For additional commands, e-mail: opensuse-buildservice+help@xxxxxxxxxxxx


--
To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@xxxxxxxxxxxx
For additional commands, e-mail: opensuse-buildservice+help@xxxxxxxxxxxx

< Previous Next >
Follow Ups
References