#!/bin/bash
set -e
unset LANG
unset ${!LC_*}
export TZ=UTC
prune_stale_files=''
# bash in sle12 has trouble with unbound variables..
if test $# -gt 0
then
	case "$1" in
		--delete) prune_stale_files='prune_stale_files' ;;
	esac
fi
#
perl_helper='/smt.repo-metadata.pl'
rsync_logfile='/smt-tumbleweed-rsync.log'
remote_rsync_module='provo-mirror.opensuse.org::opensuse'
remote_rsync_directory='/tumbleweed/repo'
remote_mirror_location="${remote_rsync_module}${remote_rsync_directory}"
local_mirror_tumbleweed='/tumbleweed'
local_cache_tumbleweed=~/tumbleweed-cache
local_cache_checksums=~/tumbleweed-cache.md5
local_prune_stale_files=~/tumbleweed-cache.prune_stale_files
#
# if this list is modified, removed repositories will disappear from our mirror
repositories=(
non-oss
debug
oss
)
repomd_files=(
CHECKSUMS
CHECKSUMS.asc
repodata/repomd.xml
repodata/repomd.xml.asc
repodata/repomd.xml.key
)
#
read user < <(id -un)
if test "${user}" != 'smt'
then
	echo "$0 needs to run as user smt, instead of '${user}'"
	exit 1
fi
# check if output directory is writeable
pushd "${local_mirror_tumbleweed}" > /dev/null
touch .
popd > /dev/null
touch "${rsync_logfile}"
test -e "${local_cache_tumbleweed}" || mkdir "$_"
# working directory
pushd "${local_cache_tumbleweed}" > /dev/null
#
read td < <(mktemp --directory --tmpdir=/dev/shm .XXX)
trap "rm -rf '${td}'" EXIT
#
t="${td}/.t"
tmp_cache_checksums="${td}/.tmp_cache_checksums"
rsync_files_from="${td}/.rsync_files_from"
repo_metadata="${td}/.repo_metadata"
repo_referenced_files="${td}/.repo_referenced_files"
repo_referenced_primary="${td}/.repo_referenced_primary"
global_referenced_metadata="${td}/.global_referenced_metadata"
global_referenced_primaries="${td}/.global_referenced_primaries"
global_referenced_packages="${td}/.global_referenced_packages"
#
#
# temporary, just to verify integrity of primary metadata
pushd "${td}" > /dev/null
mkdir -m 0700 '.gnupg'
read GNUPGHOME < <(readlink -f '.gnupg')
export GNUPGHOME
popd > /dev/null
#
do_rsync() {
	rsync \
	--archive \
	--copy-dirlinks \
	--log-file="${rsync_logfile}" \
	--no-motd \
	--sparse \
	"$@"
}
#
do_gpg() {
	local rc=1

	if gpg "$@" > /dev/null 2> "${t}"
	then
		rc=0
	fi

	# this output stderr can only be avoided by creating a temporary key
	# then sign the external key with the new private key
	# there will be still output on stderr afterwards...
	sed '
	/^gpg: Signature made /d
	/^gpg:[[:blank:]]\+using RSA key B88B2FD43DBDC284$/d
	/^gpg: Good signature from /d
	/^gpg: WARNING: This key is not certified with a trusted signature.$/d
	/^gpg:[[:blank:]]\+There is no indication that the signature belongs to the owner./d
	/^Primary key fingerprint: /d
	' "${t}" 1>&2

	return ${rc}
}
#
remove_unwanted_architectures() {
	local filelist=$1

	sed -i '
	/aarch64\//d
	/armv6l\//d
	/armv7l\//d
	/i.86\//d
	/ppc64\//d
	/ppc64le\//d
	/ppc\//d
	/riscv64\//d
	/s390\//d
	/s390x\//d
	' "${filelist}"
}
#
consistent_mirror() {
	local t_repo="${local_cache_tumbleweed}"
	local l_repo="${local_mirror_tumbleweed}"
	local r_repo="${remote_mirror_location}"
	local repository
	local metadata_file

	if test -n "${prune_stale_files}"
	then
		# prune files during next successful sync
		>> "${local_prune_stale_files}"
	fi
	# create list of primary metadata for all repositories
	for repository in "${repositories[@]}"
	do
		for metadata_file in "${repomd_files[@]}"
		do
			echo "${repository}/${metadata_file}"
		done
	done | sort -u > "${repo_metadata}"

	# cache rsync primary metadata
	do_rsync \
		"--files-from=${repo_metadata}" \
		"${r_repo}" \
		"${t_repo}" \
		|| return 1

	# verify primary metadata, nothing to do if they match
	md5sum \
		--check \
		--status \
		--strict \
		"${local_cache_checksums}" \
		&& return 0

	for repository in "${repositories[@]}"
	do
		# import repo key
		do_gpg --quiet \
			--import \
			"${repository}/repodata/repomd.xml.key" > /dev/null \
			|| return 1

		# verify repo metadata
		do_gpg --quiet \
			--verify \
			"${repository}/repodata/repomd.xml.asc" \
			"${repository}/repodata/repomd.xml" > /dev/null \
			|| return 1
		do_gpg --quiet \
			--verify \
			"${repository}/CHECKSUMS.asc" \
			"${repository}/CHECKSUMS" > /dev/null \
			|| return 1

		# prepare cache of inst-sys
		sed -n "
			/\.\(bin\|c32\|cfg\|dat\|efi\|fnt\|hlp\|jpg\|mod\|mo\|pf2\|png\|rtf\|tlk\|tr\|txt\)$/d
			/\(bootlogo\|memtest\|message\)/d
			/\/yast2-trans-/d
			/-fonts\.rpm$/d
			s@^[0-9a-f]\+[[:blank:]]\+@@
			s@^@${repository}/@
			p
		" "${repository}/CHECKSUMS" >> "${global_referenced_metadata}"

		# extract list of repomd metadata files
		"${perl_helper}" '--parse-repomd' \
			"${repository}/repodata/repomd.xml" \
			"${repo_referenced_files}" \
			"${repo_referenced_primary}" \
			|| return 1

		# put repomd metadata files into the global download list
		sed "s@^@${repository}/@" \
			"${repo_referenced_files}" \
			>> "${global_referenced_metadata}"
		sed "s@^@${repository}/@" \
			"${repo_referenced_primary}" \
			>> "${global_referenced_primaries}"
	done

	remove_unwanted_architectures "${global_referenced_metadata}" || return 1

	# cache metadata with package list, and also inst-sys
	# prune files which do not exist anymore
	cat \
		"${repo_metadata}" \
		"${global_referenced_metadata}" \
		> "${rsync_files_from}"

	do_rsync \
		"--files-from=${rsync_files_from}" \
		"${r_repo}/" \
		"${t_repo}" \
		|| return 1

	do_rsync \
		'--prune-empty-dirs' \
		'--delete-excluded' \
		'--include=/**/' \
		"--include-from=${rsync_files_from}" \
		'--exclude=*' \
		"${r_repo}/" \
		"${t_repo}" \
		|| return 1

	# verify downloaded cache of metadata
	for repository in "${repositories[@]}"
	do
		pushd "${repository}" > /dev/null

			# verify inst-sys
			sha256sum \
				--check \
				--quiet \
				--ignore-missing \
				--strict \
				'CHECKSUMS' \
				|| return 1

		popd > /dev/null
	done

	# extracting rpm list uses relative paths
	while read
	do
		set -- ${REPLY/\// }
		repository=$1
		repo_referenced_primary=$2
		pushd "${repository}" > /dev/null

			echo "${repo_referenced_primary}" > "${t}"
			# extract list of rpm files
			"${perl_helper}" '--parse-primary' \
				"${t}" \
				"${repo_referenced_files}" \
				|| return 1
			sed "s@^@${repository}/@" \
				"${repo_referenced_files}" \
				>> "${global_referenced_packages}"

		popd > /dev/null
	done < "${global_referenced_primaries}"

	remove_unwanted_architectures "${global_referenced_packages}" || return 1

	# download all packages
	do_rsync \
		"--files-from=${global_referenced_packages}" \
		"${r_repo}" \
		"${l_repo}" \
		|| return 1

	# make metadata and inst-sys visible
	cat \
		"${repo_metadata}" \
		"${global_referenced_metadata}" \
		> "${rsync_files_from}"
	do_rsync \
		"--files-from=${rsync_files_from}" \
		'--delay-updates' \
		"${t_repo}" \
		"${l_repo}" \
		|| return 1

	if test -f "${local_prune_stale_files}"
	then
		rm -fv "${local_prune_stale_files}"
		# prune files which do not exist anymore
		cat \
			"${repo_metadata}" \
			"${global_referenced_metadata}" \
			"${global_referenced_packages}" \
			> "${rsync_files_from}"
		do_rsync \
			'--ignore-existing' \
			'--prune-empty-dirs' \
			'--delete-excluded' \
			'--include=/**/' \
			"--include-from=${rsync_files_from}" \
			'--exclude=*' \
			"${r_repo}/" \
			"${l_repo}" \
			|| return 1
	fi

	# create checksum for next run
	md5sum \
		`sort -u "${repo_metadata}"` \
		> "${tmp_cache_checksums}" \
		|| return 1

	mv -f "${tmp_cache_checksums}" "${local_cache_checksums}" || return 1

	# now the mirror should be consistent
	return 0
}

if consistent_mirror
then
	echo >&1 "${0##*/} succeeded"
else
	echo >&2 "${0##*/} failed"
fi