From effe07d4f90fc387d7bdd42db1507400e33c819f Mon Sep 17 00:00:00 2001 From: Yi Yang Date: Fri, 2 Apr 2010 14:57:59 +0800 Subject: [PATCH] Support remote repos

--- build | 20 +++++++++++++++--- createrepomddeps | 24 +++++++++++++++++++++ init_buildsystem | 61 ++++++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 95 insertions(+), 10 deletions(-) diff --git a/build b/build index 18f9622..b774621 100755 --- a/build +++ b/build @@ -130,6 +130,7 @@ Known Parameters: repository NAME. The repo must be refreshed with zypp so package meta data is available locally. With emtpy NAME all enabled repositories are used. + a url can specify a remote repo. --rpms path1:path2:... Specify path where to find the RPMs for the build system @@ -145,6 +146,13 @@ Known Parameters: --root rootdir Use 'rootdir' to setup chroot environment + --cachedir cachedir + Use 'cachedir' to cache remote repo's packages, the + default cache dir is /var/cache/build, every repo + given by --repository corresponds to a subdir named + as md5sum of its repo url, forx eaxmple: + /var/cache/build/3e8ea9b47808629414a0cebc33ea285e + --oldpackages oldpackagesdir Define a directory with a former build @@ -642,6 +650,10 @@ while test -n "$1"; do BUILD_ROOT="$ARG" shift ;; + *-cachedir) + CACHE_DIR="$ARG" + shift + ;; *-oldpackages) OLD_PACKAGES="$ARG" shift @@ -783,6 +795,8 @@ while test -n "$1"; do esac done +test -z "$CACHE_DIR" && CACHE_DIR=/var/cache/build + if test -n "$KILL" ; then test -z "$SRCDIR" || usage if test -n "$VM_IMAGE" -a -n "$VM_SWAP" -a -n "$VM_TYPE"; then @@ -889,7 +903,7 @@ if test -n "$LIST_STATE" ; then } for SPECFILE in $BUILD_ROOT/usr/src/packages/SPECS/*.spec ; do : ; done fi - init_buildsystem --list-state "${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS + init_buildsystem --cachedir "$CACHE_DIR" --list-state "${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS ERR=$? rm -rf $BUILD_ROOT cleanup_and_exit $ERR @@ -1070,7 +1084,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do if test "$DO_INIT" = true ; then # do fist stage of init_buildsystem rm -f $BUILD_ROOT/.build.success - set -- init_buildsystem --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS + set -- init_buildsystem --cachedir "$CACHE_DIR" --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS echo "$* ..." "$@" || cleanup_and_exit 1 check_exit @@ -1284,7 +1298,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do echo "BUILD_INCARNATION=$INCARNATION" > $BUILD_ROOT/.buildenv CREATE_BUILD_BINARIES= egrep '^#[ ]*needsbinariesforbuild[ ]*$' >/dev/null <$MYSRCDIR/$SPECFILE && CREATE_BUILD_BINARIES=--create-build-binaries - set -- init_buildsystem "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS + set -- init_buildsystem --cachedir "$CACHE_DIR" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS echo "$* ..." "$@" || cleanup_and_exit 1 check_exit diff --git a/createrepomddeps b/createrepomddeps index 61c604a..fb2f499 100755 --- a/createrepomddeps +++ b/createrepomddeps @@ -9,6 +9,11 @@ use XML::Parser; use Data::Dumper; use Getopt::Long; use Build::Rpm; +use Digest::MD5 qw(md5 md5_hex md5_base64); +use File::Path qw(mkpath rmtree); +use File::Basename; +use LWP::UserAgent; +use URI; Getopt::Long::Configure("no_ignore_case"); my @parent = []; @@ -342,6 +347,16 @@ sub printasbuildcachefile(@) } } +sub getmetadata +{ + my $url = $_[0]; + my $dir = $_[1]; + + my $dest = $dir . "repodata"; + mkpath($dest); + system($INC[0].'/download', $dest, $url . "repodata/repomd.xml"); +} + ### main GetOptions ( @@ -378,6 +393,12 @@ for my $url (@ARGV) { } else { die "unsupported repo type: $type\n"; } + } elsif ($url =~ /^http:\/\/([^\/]*)\/?/) { + my $repoid = md5_hex($url); + my $cachedir = ($::ENV{'CACHE_DIR'} || '/var/cache/build'); + $dir = "$cachedir/$repoid/"; + getmetadata($url, $dir); + $baseurl = $url; } else { $dir = $url; $dir .= '/' unless $dir =~ /\/$/; @@ -395,6 +416,9 @@ for my $url (@ARGV) { @cursor = ([undef, $primaryparser]); my $u = $dir . $f->{'location'}; + if ($url =~ /^http:\/\/([^\/]*)\/?/) { + system($INC[0].'/download', $dir . "repodata/", $baseurl . "repodata/" . basename($u)); + } $u = 'gzip -cd ' . $u . '|' if ($u =~ /\.gz$/); # XXX my $fh; diff --git a/init_buildsystem b/init_buildsystem index efa9712..2272585 100755 --- a/init_buildsystem +++ b/init_buildsystem @@ -76,6 +76,11 @@ while test -n "$1" ; do CLEAN_BUILD="$1" shift ;; + --cachedir) + shift + CACHE_DIR="$1" + shift + ;; *) break ;; @@ -285,6 +290,9 @@ function validate_cache_file { test -z "$SRC" && SRC=`pwd` if [ "${SRC#zypp://}" != "$SRC" ]; then set -- $BUILD_DIR/createrepomddeps "$SRC" + elif [ "${SRC#http://}" != "$SRC" ]; then + mkdir -p "$(getcachedir "$SRC")" + set -- env CACHE_DIR="$CACHE_DIR" $BUILD_DIR/createrepomddeps "$SRC" elif [ ! -e "$SRC" ]; then echo "*** $SRC does not exist" >&2 cleanup_and_exit 1 @@ -336,13 +344,26 @@ fail_exit() } # modify $SRC -download_zypp() +downloadpkg() { local url="$1" - local zd="/var/cache/zypp/packages/" - SRC=$zd${url#zypp://} + local cachedir + + if [ "${url:0:7}" == "zypp://" ] ; then + cachedir="/var/cache/zypp/packages/" + SRC="$cachedir${url#zypp://}" + mkdir -p "$(dirname $SRC)" + elif [ "${url:0:7}" == "http://" ] ; then + cachedir="$(getcachedir "$url")" + local name="$(basename "$url")" + SRC="$cachedir/$name" + else + echo "Invalid url: $url" + cleanup_and_exit 1 + fi + if [ ! -e "$SRC" ]; then - local destdir="$zd/tmp" + local destdir="$cachedir/tmp" mkdir -p "$destdir" echo "downloading $url ... "; $BUILD_DIR/download "$destdir" "$url" || cleanup_and_exit 1 @@ -361,6 +382,27 @@ download_zypp() fi } +getcachedir() +{ + url=$1 + for repo in "${repos[@]}" ; do + if [ "${url:0:${#repo}}" == "$repo" ] ; then + read repoid dummy < <(echo -n "$repo" | md5sum) + echo "$CACHE_DIR/$repoid" + break + fi + done +} + +get_pkg_filename() +{ + local url="$1" + local name=$(basename $url) + local cachedir=$(getcachedir $url) + local destfile="$cachedir/$name" + echo $destfile +} + set_build_arch trap fail_exit EXIT @@ -494,8 +536,8 @@ else if [ "${SRC#/}" = "$SRC" ]; then url="$SRC" case "$url" in - zypp://*) - download_zypp "$url" + zypp://* | http://*) + downloadpkg "$url" ;; *) echo "unsupported url: $url" >&2 @@ -503,7 +545,12 @@ else ;; esac fi - ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + if [ "${SRC:0:7}" == "http://" ] ; then + target_file=$(get_pkg_filename "$SRC") + ln -s "$target_file" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + else + ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}" + fi PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL $PKG" done < $RPMLIST -- To unsubscribe, e-mail: opensuse-buildservice+unsubscribe@opensuse.org For additional commands, e-mail: opensuse-buildservice+help@opensuse.org