Mailinglist Archive: opensuse-buildservice (348 mails)

< Previous Next >
[opensuse-buildservice] [PATCH 1/1 for build] Support remote repos (v2)
  • From: Yi Yang <yi.y.yang@xxxxxxxxx>
  • Date: Fri, 2 Apr 2010 15:13:54 +0800
  • Message-id: <20100402071354.GA20225@xxxxxxxxxxxxxxxxxxxxxxx>
From effe07d4f90fc387d7bdd42db1507400e33c819f Mon Sep 17 00:00:00 2001
From: Yi Yang <yi.y.yang@xxxxxxxxx>
Date: Fri, 2 Apr 2010 14:57:59 +0800
Subject: [PATCH] Support remote repos

---
build | 20 +++++++++++++++---
createrepomddeps | 24 +++++++++++++++++++++
init_buildsystem | 61 ++++++++++++++++++++++++++++++++++++++++++++++++-------
3 files changed, 95 insertions(+), 10 deletions(-)

diff --git a/build b/build
index 18f9622..b774621 100755
--- a/build
+++ b/build
@@ -130,6 +130,7 @@ Known Parameters:
repository NAME. The repo must be refreshed with zypp
so package meta data is available locally. With emtpy
NAME all enabled repositories are used.
+ a url can specify a remote repo.

--rpms path1:path2:...
Specify path where to find the RPMs for the build system
@@ -145,6 +146,13 @@ Known Parameters:
--root rootdir
Use 'rootdir' to setup chroot environment

+ --cachedir cachedir
+ Use 'cachedir' to cache remote repo's packages, the
+ default cache dir is /var/cache/build, every repo
+ given by --repository corresponds to a subdir named
+ as md5sum of its repo url, forx eaxmple:
+ /var/cache/build/3e8ea9b47808629414a0cebc33ea285e
+
--oldpackages oldpackagesdir
Define a directory with a former build

@@ -642,6 +650,10 @@ while test -n "$1"; do
BUILD_ROOT="$ARG"
shift
;;
+ *-cachedir)
+ CACHE_DIR="$ARG"
+ shift
+ ;;
*-oldpackages)
OLD_PACKAGES="$ARG"
shift
@@ -783,6 +795,8 @@ while test -n "$1"; do
esac
done

+test -z "$CACHE_DIR" && CACHE_DIR=/var/cache/build
+
if test -n "$KILL" ; then
test -z "$SRCDIR" || usage
if test -n "$VM_IMAGE" -a -n "$VM_SWAP" -a -n "$VM_TYPE"; then
@@ -889,7 +903,7 @@ if test -n "$LIST_STATE" ; then
}
for SPECFILE in $BUILD_ROOT/usr/src/packages/SPECS/*.spec ; do : ; done
fi
- init_buildsystem --list-state "${definesnstuff[@]}" "${repos[@]}"
$USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS
+ init_buildsystem --cachedir "$CACHE_DIR" --list-state
"${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE
$BUILD_EXTRA_PACKS
ERR=$?
rm -rf $BUILD_ROOT
cleanup_and_exit $ERR
@@ -1070,7 +1084,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do
if test "$DO_INIT" = true ; then
# do fist stage of init_buildsystem
rm -f $BUILD_ROOT/.build.success
- set -- init_buildsystem --prepare "${definesnstuff[@]}"
"${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE"
$ADDITIONAL_PACKS
+ set -- init_buildsystem --cachedir "$CACHE_DIR" --prepare
"${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST
"$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS
echo "$* ..."
"$@" || cleanup_and_exit 1
check_exit
@@ -1284,7 +1298,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do
echo "BUILD_INCARNATION=$INCARNATION" > $BUILD_ROOT/.buildenv
CREATE_BUILD_BINARIES=
egrep '^#[ ]*needsbinariesforbuild[ ]*$' >/dev/null
<$MYSRCDIR/$SPECFILE && CREATE_BUILD_BINARIES=--create-build-binaries
- set -- init_buildsystem "${definesnstuff[@]}" "${repos[@]}"
$CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST
"$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS
+ set -- init_buildsystem --cachedir "$CACHE_DIR" "${definesnstuff[@]}"
"${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST
"$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS
echo "$* ..."
"$@" || cleanup_and_exit 1
check_exit
diff --git a/createrepomddeps b/createrepomddeps
index 61c604a..fb2f499 100755
--- a/createrepomddeps
+++ b/createrepomddeps
@@ -9,6 +9,11 @@ use XML::Parser;
use Data::Dumper;
use Getopt::Long;
use Build::Rpm;
+use Digest::MD5 qw(md5 md5_hex md5_base64);
+use File::Path qw(mkpath rmtree);
+use File::Basename;
+use LWP::UserAgent;
+use URI;
Getopt::Long::Configure("no_ignore_case");

my @parent = [];
@@ -342,6 +347,16 @@ sub printasbuildcachefile(@)
}
}

+sub getmetadata
+{
+ my $url = $_[0];
+ my $dir = $_[1];
+
+ my $dest = $dir . "repodata";
+ mkpath($dest);
+ system($INC[0].'/download', $dest, $url . "repodata/repomd.xml");
+}
+
### main

GetOptions (
@@ -378,6 +393,12 @@ for my $url (@ARGV) {
} else {
die "unsupported repo type: $type\n";
}
+ } elsif ($url =~ /^http:\/\/([^\/]*)\/?/) {
+ my $repoid = md5_hex($url);
+ my $cachedir = ($::ENV{'CACHE_DIR'} || '/var/cache/build');
+ $dir = "$cachedir/$repoid/";
+ getmetadata($url, $dir);
+ $baseurl = $url;
} else {
$dir = $url;
$dir .= '/' unless $dir =~ /\/$/;
@@ -395,6 +416,9 @@ for my $url (@ARGV) {
@cursor = ([undef, $primaryparser]);

my $u = $dir . $f->{'location'};
+ if ($url =~ /^http:\/\/([^\/]*)\/?/) {
+ system($INC[0].'/download', $dir . "repodata/", $baseurl . "repodata/"
. basename($u));
+ }
$u = 'gzip -cd ' . $u . '|' if ($u =~ /\.gz$/); # XXX

my $fh;
diff --git a/init_buildsystem b/init_buildsystem
index efa9712..2272585 100755
--- a/init_buildsystem
+++ b/init_buildsystem
@@ -76,6 +76,11 @@ while test -n "$1" ; do
CLEAN_BUILD="$1"
shift
;;
+ --cachedir)
+ shift
+ CACHE_DIR="$1"
+ shift
+ ;;
*)
break
;;
@@ -285,6 +290,9 @@ function validate_cache_file {
test -z "$SRC" && SRC=`pwd`
if [ "${SRC#zypp://}" != "$SRC" ]; then
set -- $BUILD_DIR/createrepomddeps "$SRC"
+ elif [ "${SRC#http://}"; != "$SRC" ]; then
+ mkdir -p "$(getcachedir "$SRC")"
+ set -- env CACHE_DIR="$CACHE_DIR" $BUILD_DIR/createrepomddeps
"$SRC"
elif [ ! -e "$SRC" ]; then
echo "*** $SRC does not exist" >&2
cleanup_and_exit 1
@@ -336,13 +344,26 @@ fail_exit()
}

# modify $SRC
-download_zypp()
+downloadpkg()
{
local url="$1"
- local zd="/var/cache/zypp/packages/"
- SRC=$zd${url#zypp://}
+ local cachedir
+
+ if [ "${url:0:7}" == "zypp://" ] ; then
+ cachedir="/var/cache/zypp/packages/"
+ SRC="$cachedir${url#zypp://}"
+ mkdir -p "$(dirname $SRC)"
+ elif [ "${url:0:7}" == "http://"; ] ; then
+ cachedir="$(getcachedir "$url")"
+ local name="$(basename "$url")"
+ SRC="$cachedir/$name"
+ else
+ echo "Invalid url: $url"
+ cleanup_and_exit 1
+ fi
+
if [ ! -e "$SRC" ]; then
- local destdir="$zd/tmp"
+ local destdir="$cachedir/tmp"
mkdir -p "$destdir"
echo "downloading $url ... ";
$BUILD_DIR/download "$destdir" "$url" || cleanup_and_exit 1
@@ -361,6 +382,27 @@ download_zypp()
fi
}

+getcachedir()
+{
+ url=$1
+ for repo in "${repos[@]}" ; do
+ if [ "${url:0:${#repo}}" == "$repo" ] ; then
+ read repoid dummy < <(echo -n "$repo" | md5sum)
+ echo "$CACHE_DIR/$repoid"
+ break
+ fi
+ done
+}
+
+get_pkg_filename()
+{
+ local url="$1"
+ local name=$(basename $url)
+ local cachedir=$(getcachedir $url)
+ local destfile="$cachedir/$name"
+ echo $destfile
+}
+
set_build_arch

trap fail_exit EXIT
@@ -494,8 +536,8 @@ else
if [ "${SRC#/}" = "$SRC" ]; then
url="$SRC"
case "$url" in
- zypp://*)
- download_zypp "$url"
+ zypp://* | http://*)
+ downloadpkg "$url"
;;
*)
echo "unsupported url: $url" >&2
@@ -503,7 +545,12 @@ else
;;
esac
fi
- ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}"
+ if [ "${SRC:0:7}" == "http://"; ] ; then
+ target_file=$(get_pkg_filename "$SRC")
+ ln -s "$target_file"
"$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}"
+ else
+ ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}"
+ fi
PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL $PKG"
done < $RPMLIST

--
To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@xxxxxxxxxxxx
For additional commands, e-mail: opensuse-buildservice+help@xxxxxxxxxxxx

< Previous Next >
Follow Ups