commit google-compute-engine for openSUSE:Factory

Hello community, here is the log from the commit of package google-compute-engine for openSUSE:Factory checked in at 2019-06-01 09:57:00 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/google-compute-engine (Old) and /work/SRC/openSUSE:Factory/.google-compute-engine.new.5148 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "google-compute-engine" Sat Jun 1 09:57:00 2019 rev:15 rq:706513 version:20190522 Changes: -------- --- /work/SRC/openSUSE:Factory/google-compute-engine/google-compute-engine.changes 2019-05-17 23:43:51.817889733 +0200 +++ /work/SRC/openSUSE:Factory/.google-compute-engine.new.5148/google-compute-engine.changes 2019-06-01 09:57:01.643170090 +0200 @@ -1,0 +2,16 @@ +Mon May 27 10:33:55 UTC 2019 - John Paul Adrian Glaubitz + +- Update to version 20190522 (bsc#1136266, bsc#1136267) + + Google Compute Engine + * Fix guest attributes flow in Python 3. + + Google Compute Engine OS Login + * Update OS Login control file for FreeBSD support. +- from version 20190521 + + Google Compute Engine + * Retry download for metadata scripts. + * Fix script retrieval in Python 3. + * Disable boto config in Python 3. + * Update SSH host keys in guest attributes. + * Fix XPS settings with more than 64 vCPUs. + +------------------------------------------------------------------- Old: ---- compute-image-packages-20190416.tar.gz New: ---- compute-image-packages-20190522.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ google-compute-engine.spec ++++++ --- /var/tmp/diff_new_pack.yy97x4/_old 2019-06-01 09:57:02.175169909 +0200 +++ /var/tmp/diff_new_pack.yy97x4/_new 2019-06-01 09:57:02.175169909 +0200 @@ -16,10 +16,10 @@ # -%define setup_version 2.8.13 +%define setup_version 2.8.16 Name: google-compute-engine -Version: 20190416 +Version: 20190522 Release: 0 Summary: GCE Instance Initialization License: Apache-2.0 ++++++ compute-image-packages-20190416.tar.gz -> compute-image-packages-20190522.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/README.md new/compute-image-packages-20190522/README.md --- old/compute-image-packages-20190416/README.md 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/README.md 2019-05-22 22:41:34.000000000 +0200 @@ -43,10 +43,11 @@ CentOS 7 | rpm | 2.7 | systemd RHEL 6 | rpm | 2.6 | upstart RHEL 7 | rpm | 2.7 | systemd +RHEL 8 | rpm | 3.6 | systemd Ubuntu 14.04 | deb | 2.7 | upstart Ubuntu 16.04 | deb | 3.5 or 2.7 | systemd Ubuntu 18.04 | deb | 3.6 | systemd -Ubuntu 18.10 | deb | 3.6 | systemd +Ubuntu 19.04 | deb | 3.7 | systemd Debian 9 | deb | 3.5 or 2.7 | systemd We build the following packages for the Linux guest environment. @@ -123,9 +124,9 @@ ``` DIST=7 tee /etc/yum.repos.d/google-cloud.repo << EOM -[google-cloud-compute] -name=Google Cloud Compute -baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el${DIST}-x86_64 +[google-compute-engine] +name=Google Compute Engine +baseurl=https://packages.cloud.google.com/yum/repos/google-compute-engine-el${DIST}-x86_64-stable enabled=1 gpgcheck=1 repo_gpgcheck=1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/daisy_workflows/build_el.wf.json new/compute-image-packages-20190522/daisy_workflows/build_el.wf.json --- old/compute-image-packages-20190416/daisy_workflows/build_el.wf.json 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/daisy_workflows/build_el.wf.json 2019-05-22 22:41:34.000000000 +0200 @@ -21,6 +21,12 @@ "setup-disk": { "CreateDisks": [ { + "Name": "disk-el8-build", + "SourceImage": "projects/rhel-cloud/global/images/family/rhel-8", + "SizeGb": "10", + "Type": "pd-ssd" + }, + { "Name": "disk-el7-build", "SourceImage": "projects/centos-cloud/global/images/family/centos-7", "SizeGb": "10", @@ -37,6 +43,20 @@ "package-build": { "CreateInstances": [ { + "Name": "inst-el8-build", + "Disks": [ + {"Source": "disk-el8-build"} + ], + "MachineType": "n1-standard-2", + "Metadata": { + "github_branch": "${github_branch}", + "github_repo": "${github_repo}", + "output_path": "${output_path}" + }, + "Scopes": ["https://www.googleapis.com/auth/devstorage.read_write"], + "StartupScript": "build_el_packages.sh" + }, + { "Name": "inst-el7-build", "Disks": [ {"Source": "disk-el7-build"} @@ -69,6 +89,14 @@ "wait-for-build": { "WaitForInstancesSignal": [ { + "Name": "inst-el8-build", + "SerialOutput": { + "Port": 1, + "SuccessMatch": "BuildSuccess:", + "FailureMatch": "BuildFailed:" + } + }, + { "Name": "inst-el7-build", "SerialOutput": { "Port": 1, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/gce-disk-expand/packaging/gce-disk-expand.spec new/compute-image-packages-20190522/packages/gce-disk-expand/packaging/gce-disk-expand.spec --- old/compute-image-packages-20190416/packages/gce-disk-expand/packaging/gce-disk-expand.spec 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/gce-disk-expand/packaging/gce-disk-expand.spec 2019-05-22 22:41:34.000000000 +0200 @@ -33,13 +33,13 @@ %install mv src/expandfs-lib.sh src/usr/share/dracut/modules.d/50expand_rootfs/ -%if 0%{?rhel} == 7 +%if 0%{?rhel} >= 7 ./dracut6_7.sh %endif rsync -Pravz src/ %{buildroot} %files -%if 0%{?rhel} == 7 +%if 0%{?rhel} >= 7 %attr(755,root,root) /usr/lib/dracut/modules.d/50expand_rootfs/* %else %attr(755,root,root) /usr/share/dracut/modules.d/50expand_rootfs/* diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/gce-disk-expand/packaging/setup_deb.sh new/compute-image-packages-20190522/packages/gce-disk-expand/packaging/setup_deb.sh --- old/compute-image-packages-20190416/packages/gce-disk-expand/packaging/setup_deb.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/gce-disk-expand/packaging/setup_deb.sh 2019-05-22 22:41:34.000000000 +0200 @@ -26,7 +26,7 @@ sudo apt-get -y install dh-systemd # DEB creation tools. -sudo apt-get -y install debhelper devscripts build-essential +sudo apt-get -y install debhelper devscripts build-essential rsync rm -rf /tmp/debpackage mkdir /tmp/debpackage diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/gce-disk-expand/packaging/setup_rpm.sh new/compute-image-packages-20190522/packages/gce-disk-expand/packaging/setup_rpm.sh --- old/compute-image-packages-20190416/packages/gce-disk-expand/packaging/setup_rpm.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/gce-disk-expand/packaging/setup_rpm.sh 2019-05-22 22:41:34.000000000 +0200 @@ -24,7 +24,7 @@ fi # RPM creation tools. -sudo yum -y install rpmdevtools +sudo yum -y install rpmdevtools rsync rm -rf ${rpm_working_dir} mkdir -p ${rpm_working_dir}/{SOURCES,SPECS} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/gce-disk-expand/src/expandfs-lib.sh new/compute-image-packages-20190522/packages/gce-disk-expand/src/expandfs-lib.sh --- old/compute-image-packages-20190416/packages/gce-disk-expand/src/expandfs-lib.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/gce-disk-expand/src/expandfs-lib.sh 2019-05-22 22:41:34.000000000 +0200 @@ -24,7 +24,7 @@ case "${fs_type}" in xfs) echo "XFS filesystems must be mounted to be resized, deferring." - expand_xfs="true" + echo "true" > /tmp/xfs_resize return 1 ;; ext*) @@ -136,6 +136,7 @@ echo "Unable to resize ${disk}${partnum}: ${out}" return 1 fi + udevadm settle } # Resizes partition by deleting and recreating with end position. @@ -184,4 +185,5 @@ return 1 fi done + udevadm settle ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs.sh new/compute-image-packages-20190522/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs.sh --- old/compute-image-packages-20190416/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs.sh 2019-05-22 22:41:34.000000000 +0200 @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -expand_xfs="" # Global to message next script. - # Contains dracut-specific logic for detecting disk, then calls appropriate # library functions. main() { @@ -26,11 +24,7 @@ return fi - # Wait for any of the initial udev events to finish otherwise growpart - # might fail. - udevsettle - - if ! out=$(get_partition "$rootdev"); then + if ! out=$(split_partition "$rootdev"); then echo "Failed to detect disk and partition info: ${out}" return fi @@ -63,8 +57,6 @@ fi fi - udevsettle - if ! out=$(resize_filesystem "$rootdev"); then echo "Failed to resize filesystem: ${out}" return diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs_dummy.sh new/compute-image-packages-20190522/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs_dummy.sh --- old/compute-image-packages-20190416/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs_dummy.sh 1970-01-01 01:00:00.000000000 +0100 +++ new/compute-image-packages-20190522/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs_dummy.sh 2019-05-22 22:41:34.000000000 +0200 @@ -0,0 +1,3 @@ +#!/bin/sh +# Dummy script, to make sure systemd executes the cmdline stage (which exports +# the 'root' variable required for expand_root) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/install new/compute-image-packages-20190522/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/install --- old/compute-image-packages-20190416/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/install 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/install 2019-05-22 22:41:34.000000000 +0200 @@ -16,8 +16,9 @@ # Dracut install script for RHEL/CentOS inst "$moddir/expandfs-lib.sh" "/lib/expandfs-lib.sh" +inst_hook cmdline 50 "$moddir/expand_rootfs_dummy.sh" inst_hook pre-mount 50 "$moddir/expand_rootfs.sh" -inst_hook pre-pivot 50 "$moddir/xfs_growfs.sh" +inst_hook pre-pivot 99 "$moddir/xfs_growfs.sh" dracut_install parted dracut_install cut @@ -25,4 +26,5 @@ dracut_install grep dracut_install resize2fs dracut_install e2fsck +dracut_install udevadm dracut_install -o xfs_growfs diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/xfs_growfs.sh new/compute-image-packages-20190522/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/xfs_growfs.sh --- old/compute-image-packages-20190416/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/xfs_growfs.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/xfs_growfs.sh 2019-05-22 22:41:34.000000000 +0200 @@ -14,7 +14,7 @@ # limitations under the License. main() { - if [ -z "$expand_xfs" ]; then + if [ ! -e /tmp/xfs_resize ]; then return fi @@ -23,11 +23,18 @@ return fi if xfs_growfs -d -n /sysroot; then + echo "Mounting filesystem rw." + if ! $(mount -o rw,remount /sysroot); then + echo "Remount failed." + return + fi echo "Resizing XFS filesystem" if ! out=$(xfs_growfs -d /sysroot); then echo "Failed to resize: ${out}" + mount -o ro,remount /sysroot return fi + mount -o ro,remount /sysroot fi } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine/packaging/debian/changelog new/compute-image-packages-20190522/packages/google-compute-engine/packaging/debian/changelog --- old/compute-image-packages-20190416/packages/google-compute-engine/packaging/debian/changelog 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine/packaging/debian/changelog 2019-05-22 22:41:34.000000000 +0200 @@ -1,3 +1,15 @@ +google-compute-engine (2.8.16-1) stable; urgency=low + + * Fix instance setup in Python 3 environments. + + -- Google Cloud Team Tue, 22 May 2019 12:00:00 -0700 + +google-compute-engine (2.8.15-1) stable; urgency=low + + * Fix XPS settings with more than 64 vCPUs. + + -- Google Cloud Team Tue, 21 May 2019 12:00:00 -0700 + google-compute-engine (2.8.14-1) stable; urgency=low * Upstart systems: only run startup scripts at boot. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine/packaging/google-compute-engine.spec new/compute-image-packages-20190522/packages/google-compute-engine/packaging/google-compute-engine.spec --- old/compute-image-packages-20190416/packages/google-compute-engine/packaging/google-compute-engine.spec 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine/packaging/google-compute-engine.spec 2019-05-22 22:41:34.000000000 +0200 @@ -12,16 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +# For EL7, if building on CentOS, override dist to be el7. +%if 0%{?rhel} == 7 + %define dist .el7 +%endif + Name: google-compute-engine Version: %{_version} -Release: 1.el7 +Release: 1%{?dist} Summary: Google Compute Engine guest environment. License: ASL 2.0 Url: https://github.com/GoogleCloudPlatform/compute-image-packages Source0: %{name}_%{version}.orig.tar.gz Requires: curl Requires: google-compute-engine-oslogin +%if 0%{?rhel} == 8 +Requires: python3-google-compute-engine = %{version} +%else Requires: python-google-compute-engine = %{version} +%endif Requires: rsyslog BuildArch: noarch diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine/packaging/setup_deb.sh new/compute-image-packages-20190522/packages/google-compute-engine/packaging/setup_deb.sh --- old/compute-image-packages-20190416/packages/google-compute-engine/packaging/setup_deb.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine/packaging/setup_deb.sh 2019-05-22 22:41:34.000000000 +0200 @@ -14,7 +14,7 @@ # limitations under the License. NAME="google-compute-engine" -VERSION="2.8.14" +VERSION="2.8.16" working_dir=${PWD} if [[ $(basename "$working_dir") != $NAME ]]; then diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine/packaging/setup_rpm.sh new/compute-image-packages-20190522/packages/google-compute-engine/packaging/setup_rpm.sh --- old/compute-image-packages-20190416/packages/google-compute-engine/packaging/setup_rpm.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine/packaging/setup_rpm.sh 2019-05-22 22:41:34.000000000 +0200 @@ -14,7 +14,7 @@ # limitations under the License. NAME="google-compute-engine" -VERSION="2.8.14" +VERSION="2.8.16" rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION} working_dir=${PWD} @@ -23,10 +23,6 @@ exit 1 fi -# Build dependencies. -sudo yum -y install make gcc-c++ libcurl-devel json-c json-c-devel pam-devel \ - policycoreutils-python boost-devel - # RPM creation tools. sudo yum -y install rpmdevtools diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine/src/lib/systemd/system/google-accounts-daemon.service new/compute-image-packages-20190522/packages/google-compute-engine/src/lib/systemd/system/google-accounts-daemon.service --- old/compute-image-packages-20190416/packages/google-compute-engine/src/lib/systemd/system/google-accounts-daemon.service 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine/src/lib/systemd/system/google-accounts-daemon.service 2019-05-22 22:41:34.000000000 +0200 @@ -7,6 +7,7 @@ Type=simple ExecStart=/usr/bin/google_accounts_daemon OOMScoreAdjust=-999 +Restart=always [Install] WantedBy=multi-user.target diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine/src/usr/bin/google_set_multiqueue new/compute-image-packages-20190522/packages/google-compute-engine/src/usr/bin/google_set_multiqueue --- old/compute-image-packages-20190416/packages/google-compute-engine/src/usr/bin/google_set_multiqueue 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine/src/usr/bin/google_set_multiqueue 2019-05-22 22:41:34.000000000 +0200 @@ -125,10 +125,16 @@ xps=$((xps | (1 << cpu))) done - # Linux xps_cpus requires a hex number with commas every 32 bits. - # It ignores all bits above # cpus, so unconditionally write a - # 64 bit hex value, with a comma between dwords. - xps_string=`printf "%08x,%08x" $((xps >> 32 & 0xffffffff)) $((xps & 0xffffffff))` + # Linux xps_cpus requires a hex number with commas every 32 bits. It ignores + # all bits above # cpus, so write a list of comma separated 32 bit hex values + # with a comma between dwords. + xps_dwords=() + for i in $(seq 0 $(((num_cpus - 1) / 32))) + do + xps_dwords+=(`printf "%08x" $((xps & 0xffffffff))`) + done + xps_string=$(IFS=, ; echo "${xps_dwords[*]}") + echo ${xps_string} > $q printf "Queue %d XPS=%s for %s\n" $queue_num `cat $q` $q diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/Makefile new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/Makefile --- old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/Makefile 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/Makefile 2019-05-22 22:41:34.000000000 +0200 @@ -4,7 +4,7 @@ NAME = google-compute-engine-$(BASENAME) MAJOR = 1 MINOR = 5 -REVISION = 2 +REVISION = 3 LIBNSS_CACHE_OSLOGIN = libnss_cache_$(BASENAME) LIBNSS_CACHE_OSLOGIN_NAME = libnss_cache_$(NAME)-$(MAJOR).$(MINOR).$(REVISION).so diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/bin/google_oslogin_control new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/bin/google_oslogin_control --- old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/bin/google_oslogin_control 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/bin/google_oslogin_control 2019-05-22 22:41:34.000000000 +0200 @@ -13,6 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +is_freebsd() { + [ "$(uname)" = "FreeBSD" ] + return $? +} + nss_config="/etc/nsswitch.conf" pam_sshd_config="/etc/pam.d/sshd" pam_su_config="/etc/pam.d/su" @@ -20,15 +25,13 @@ group_config="/etc/security/group.conf" sudoers_dir="/var/google-sudoers.d" users_dir="/var/google-users.d" -sudoers_file="/etc/sudoers.d/google-oslogin" added_comment="# Added by Google Compute Engine OS Login." sshd_block="#### Google OS Login control. Do not edit this section. ####" sshd_end_block="#### End Google OS Login control section. ####" - -is_freebsd() { - [ "$(uname)" = "FreeBSD" ] - return $? -} +sudoers_file="/etc/sudoers.d/google-oslogin" +if is_freebsd; then + sudoers_file="/usr/local/etc/sudoers.d/google-oslogin" +fi # Update nsswitch.conf to include OS Login NSS module for passwd. modify_nsswitch_conf() { @@ -97,7 +100,7 @@ if ! grep -q "$sshd_block" "$sshd_config"; then # Remove old-style additions. $sed -i"" "/${added_comment}/,+1d" "$sshd_config" - /bin/echo -e "\n\n${sshd_block}\n${sshd_end_block}" >> "$sshd_config" + printf "\n\n${sshd_block}\n${sshd_end_block}" >> "$sshd_config" fi for entry in "$sshd_auth_keys_command" "$sshd_auth_keys_command_user"; do diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/packaging/debian/changelog new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/packaging/debian/changelog --- old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/packaging/debian/changelog 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/packaging/debian/changelog 2019-05-22 22:41:34.000000000 +0200 @@ -1,3 +1,9 @@ +google-compute-engine-oslogin (1.5.3-1) unstable; urgency=low + + * Update OS Login control file for FreeBSD support. + + -- Google Cloud Team Tue, 22 Nay 2019 12:00:00 -0700 + google-compute-engine-oslogin (1.5.2-1) unstable; urgency=low * Fix pam_group ordering detection. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/packaging/google-compute-engine-oslogin.spec new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/packaging/google-compute-engine-oslogin.spec --- old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/packaging/google-compute-engine-oslogin.spec 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/packaging/google-compute-engine-oslogin.spec 2019-05-22 22:41:34.000000000 +0200 @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Force the dist to be el7 to avoid el7.centos. +# For EL7, if building on CentOS, override dist to be el7. %if 0%{?rhel} == 7 %define dist .el7 %endif @@ -29,11 +29,17 @@ BuildRequires: gcc-c++ BuildRequires: make BuildRequires: libcurl -BuildRequires: json-c +BuildRequires: json-c-devel BuildRequires: pam-devel +%if 0%{?rhel} == 8 +BuildRequires: python3-policycoreutils +Requires: python3-policycoreutils +%else BuildRequires: policycoreutils-python -Requires: boost-regex Requires: policycoreutils-python +%endif +Requires: boost-regex +Requires: json-c %define pam_install_path /%{_lib}/security @@ -41,6 +47,8 @@ This package contains several libraries and changes to enable OS Login functionality for Google Compute Engine. +%global debug_package %{nil} + %prep %setup @@ -53,10 +61,14 @@ %files %doc -/%{_lib}/libnss_%{name}-%{version}.so -/%{_lib}/libnss_cache_%{name}-%{version}.so -%{pam_install_path}/pam_oslogin_admin.so -%{pam_install_path}/pam_oslogin_login.so +%attr(0755,-,-) /%{_lib}/libnss_%{name}-%{version}.so +%attr(0755,-,-) /%{_lib}/libnss_cache_%{name}-%{version}.so +%if 0%{?rhel} == 8 +/%{_lib}/libnss_oslogin.so.2 +/%{_lib}/libnss_cache_oslogin.so.2 +%endif +%attr(0755,-,-) %{pam_install_path}/pam_oslogin_admin.so +%attr(0755,-,-) %{pam_install_path}/pam_oslogin_login.so /usr/bin/google_authorized_keys /usr/bin/google_oslogin_control /usr/bin/google_oslogin_nss_cache diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/packaging/setup_deb.sh new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/packaging/setup_deb.sh --- old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/packaging/setup_deb.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/packaging/setup_deb.sh 2019-05-22 22:41:34.000000000 +0200 @@ -14,7 +14,7 @@ # limitations under the License. NAME="google-compute-engine-oslogin" -VERSION="1.5.2" +VERSION="1.5.3" working_dir=${PWD} if [[ $(basename "$working_dir") != $NAME ]]; then diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/packaging/setup_rpm.sh new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/packaging/setup_rpm.sh --- old/compute-image-packages-20190416/packages/google-compute-engine-oslogin/packaging/setup_rpm.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/google-compute-engine-oslogin/packaging/setup_rpm.sh 2019-05-22 22:41:34.000000000 +0200 @@ -15,7 +15,7 @@ NAME="google-compute-engine-oslogin" -VERSION="1.5.2" +VERSION="1.5.3" rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION} working_dir=${PWD} @@ -24,11 +24,14 @@ exit 1 fi -# Build dependencies. -sudo yum -y install make gcc-c++ libcurl-devel json-c json-c-devel pam-devel policycoreutils-python boost-devel +sudo yum -y install rpmdevtools make gcc-c++ json-c \ + libcurl-devel pam-devel boost-devel json-c-devel -# RPM creation tools. -sudo yum -y install rpmdevtools +if grep -q '^\(CentOS\|Red Hat\)[^0-9]*8\..' /etc/redhat-release; then + sudo yum -y install python3-policycoreutils +else + sudo yum -y install policycoreutils-python +fi rm -rf /tmp/rpmpackage mkdir -p ${rpm_working_dir}/{SOURCES,SPECS} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/README.md new/compute-image-packages-20190522/packages/python-google-compute-engine/README.md --- old/compute-image-packages-20190416/packages/python-google-compute-engine/README.md 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/README.md 2019-05-22 22:41:34.000000000 +0200 @@ -28,8 +28,6 @@ based authentication. * **Clock skew** daemon to keep the system clock in sync after VM start and stop events. -* **Disk expand** scripts to expand the VM root partition for CentOS 6, - CentOS 7, RHEL 6, and RHEL 7 images. * **Instance setup** scripts to execute VM configuration scripts during boot. * **Network** daemon that handles network setup for multiple network interfaces on boot and integrates network load balancing with @@ -38,7 +36,7 @@ shutdown. The Linux guest environment is written in Python and is version agnostic -between Python 2.6 and 3.5. There is complete unittest coverage for every Python +between Python 2.6 and 3.7. There is complete unittest coverage for every Python library and script. The design of various guest libraries, daemons, and scripts, are detailed in the sections below. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/boto/tests/compute_auth_test.py new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/boto/tests/compute_auth_test.py --- old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/boto/tests/compute_auth_test.py 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/boto/tests/compute_auth_test.py 2019-05-22 22:41:34.000000000 +0200 @@ -15,11 +15,16 @@ """Unittest for compute_auth.py module.""" -from google_compute_engine.boto import compute_auth +import sys + from google_compute_engine.test_compat import mock from google_compute_engine.test_compat import unittest +if sys.version_info < (3, 0): + from google_compute_engine.boto import compute_auth + +@unittest.skipIf(sys.version_info > (3, 0), 'Skipping for python3.') class ComputeAuthTest(unittest.TestCase): def setUp(self): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/compat.py new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/compat.py --- old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/compat.py 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/compat.py 2019-05-22 22:41:34.000000000 +0200 @@ -19,7 +19,7 @@ import subprocess import sys -if sys.version_info >= (3, 6): +if sys.version_info >= (3, 7): import distro else: import platform as distro diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/constants.py new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/constants.py --- old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/constants.py 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/constants.py 2019-05-22 22:41:34.000000000 +0200 @@ -16,6 +16,7 @@ """A module for global constants.""" import platform +import sys OSLOGIN_CONTROL_SCRIPT = 'google_oslogin_control' OSLOGIN_NSS_CACHE_SCRIPT = 'google_oslogin_nss_cache' @@ -41,3 +42,8 @@ OSLOGIN_NSS_CACHE = '/etc/oslogin_passwd.cache' SYSCONFDIR = '/etc/default' SYSLOG_SOCKET = '/dev/log' + +if sys.version_info >= (3, 0): + SET_BOTO_CONFIG = 'false' +else: + SET_BOTO_CONFIG = 'true' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py --- old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py 2019-05-22 22:41:34.000000000 +0200 @@ -78,7 +78,7 @@ 'host_key_types': 'ecdsa,ed25519,rsa', 'optimize_local_ssd': 'true', 'network_enabled': 'true', - 'set_boto_config': 'true', + 'set_boto_config': constants.SET_BOTO_CONFIG, 'set_host_keys': 'true', 'set_multiqueue': 'true', }, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py --- old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py 2019-05-22 22:41:34.000000000 +0200 @@ -28,9 +28,21 @@ from google_compute_engine import logger from google_compute_engine import metadata_watcher from google_compute_engine.boto import boto_config +from google_compute_engine.compat import urlerror +from google_compute_engine.compat import urlrequest from google_compute_engine.instance_setup import instance_config +class PutRequest(urlrequest.Request): + def get_method(self): + return 'PUT' + + +GUEST_ATTRIBUTES_URL = ('http://metadata.google.internal/computeMetadata/v1beta1/' + 'instance/guest-attributes') +HOSTKEY_NAMESPACE = 'hostkeys' + + class InstanceSetup(object): """Initialize the instance the first time it boots.""" @@ -122,6 +134,9 @@ Args: key_type: string, the type of the SSH key. key_dest: string, a file location to store the SSH key. + + Returns: + tuple, key_type and public key string. """ # Create a temporary file to save the created RSA keys. with tempfile.NamedTemporaryFile(prefix=key_type, delete=True) as temp: @@ -140,6 +155,29 @@ file_utils.SetPermissions(key_dest, mode=0o600) file_utils.SetPermissions('%s.pub' % key_dest, mode=0o644) + with open('%s.pub' % key_dest, 'r') as pk: + key_data = pk.read() + + key_values = key_data.split() + if len(key_values) < 2: + self.logger.warning('Could not read host key from %s.pub.', key_dest) + return + else: + return key_values[0], key_values[1] + + def _WriteHostKeyToGuestAttributes(self, key_type, key_value): + """Write a host key to guest attributes, ignoring errors.""" + headers = {'Metadata-Flavor': 'Google'} + url = '%s/%s/%s' % (GUEST_ATTRIBUTES_URL, HOSTKEY_NAMESPACE, key_type) + key_value = key_value.encode('utf-8') + req = PutRequest(url, key_value, headers) + try: + response = urlrequest.urlopen(req) + self.logger.debug(response) + self.logger.info('Wrote %s host key to guest attributes.', key_type) + except urlerror.HTTPError: + self.logger.info('Unable to write %s host key to guest attributes.', + key_type) def _StartSshd(self): """Initialize the SSH daemon.""" @@ -180,7 +218,9 @@ for key_file in set(key_files) | set(key_types_files): key_type = file_regex.match(key_file).group('type') key_dest = os.path.join(key_dir, key_file) - self._GenerateSshKey(key_type, key_dest) + key_data = self._GenerateSshKey(key_type, key_dest) + if key_data: + self._WriteHostKeyToGuestAttributes(key_data[0], key_data[1]) self._StartSshd() self.instance_config.SetOption(section, 'instance_id', str(instance_id)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py --- old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py 2019-05-22 22:41:34.000000000 +0200 @@ -18,6 +18,7 @@ import subprocess from google_compute_engine.instance_setup import instance_setup +from google_compute_engine.test_compat import builtin from google_compute_engine.test_compat import mock from google_compute_engine.test_compat import unittest @@ -231,22 +232,33 @@ temp_dest = '/tmp/dest' mock_tempfile.return_value = mock_tempfile mock_tempfile.__enter__.return_value.name = temp_dest - - instance_setup.InstanceSetup._GenerateSshKey( - self.mock_setup, key_type, key_dest) - expected_calls = [ - mock.call.tempfile(prefix=key_type, delete=True), - mock.call.tempfile.__enter__(), - mock.call.tempfile.__exit__(None, None, None), - mock.call.logger.info(mock.ANY, key_dest), - mock.call.call( - ['ssh-keygen', '-t', key_type, '-f', temp_dest, '-N', '', '-q']), - mock.call.move(temp_dest, key_dest), - mock.call.move('%s.pub' % temp_dest, '%s.pub' % key_dest), - mock.call.permissions(key_dest, mode=0o600), - mock.call.permissions('%s.pub' % key_dest, mode=0o644), - ] - self.assertEqual(mocks.mock_calls, expected_calls) + mock_open = mock.mock_open() + key_file_contents = 'ssh-rsa asdfasdf' + expected_key_data = ('ssh-rsa', 'asdfasdf') + + with mock.patch('%s.open' % builtin, mock_open, create=False): + mock_open().read.return_value = key_file_contents + key_data = instance_setup.InstanceSetup._GenerateSshKey( + self.mock_setup, key_type, key_dest) + expected_calls = [ + mock.call.tempfile(prefix=key_type, delete=True), + mock.call.tempfile.__enter__(), + mock.call.tempfile.__exit__(None, None, None), + mock.call.logger.info(mock.ANY, key_dest), + mock.call.call( + ['ssh-keygen', '-t', key_type, '-f', temp_dest, '-N', '', '-q']), + mock.call.move(temp_dest, key_dest), + mock.call.move('%s.pub' % temp_dest, '%s.pub' % key_dest), + mock.call.permissions(key_dest, mode=0o600), + mock.call.permissions('%s.pub' % key_dest, mode=0o644), + ] + self.assertEqual(mocks.mock_calls, expected_calls) + self.assertEqual(key_data, expected_key_data) + + mock_open().read.return_value = '' + key_data = instance_setup.InstanceSetup._GenerateSshKey( + self.mock_setup, key_type, key_dest) + self.assertEqual(key_data, None) @mock.patch('google_compute_engine.instance_setup.instance_setup.subprocess.check_call') def testGenerateSshKeyProcessError(self, mock_call): @@ -318,6 +330,33 @@ instance_setup.InstanceSetup._SetSshHostKeys(self.mock_setup) self.mock_instance_config.SetOption.assert_not_called() + @mock.patch('google_compute_engine.instance_setup.instance_setup.urlrequest.urlopen') + @mock.patch('google_compute_engine.instance_setup.instance_setup.PutRequest') + def testWriteHostKeyToGuestAttributes(self, mock_put, mock_urlopen): + key_type = 'ssh-rsa' + key_value = 'asdfasdf' + encoded_key_value = key_value.encode('utf-8') + expected_url = ('http://metadata.google.internal/computeMetadata/v1beta1/' + 'instance/guest-attributes/hostkeys/%s' % key_type) + headers = {'Metadata-Flavor': 'Google'} + + instance_setup.InstanceSetup._WriteHostKeyToGuestAttributes( + self.mock_setup, key_type, key_value) + self.mock_logger.info.assert_called_with( + 'Wrote %s host key to guest attributes.', key_type) + mock_put.assert_called_with(expected_url, encoded_key_value, headers) + + mock_urlopen.side_effect = instance_setup.urlerror.HTTPError( + 'http://foo', 403, 'Forbidden', {}, None) + instance_setup.InstanceSetup._WriteHostKeyToGuestAttributes( + self.mock_setup, key_type, key_value) + self.mock_logger.info.assert_called_with( + 'Unable to write %s host key to guest attributes.', key_type) + + def testPutRequest(self): + put_request = instance_setup.PutRequest('http://example.com/') + self.assertEqual(put_request.get_method(), 'PUT') + @mock.patch('google_compute_engine.instance_setup.instance_setup.os.listdir') def testSetSshHostKeysFirstBoot(self, mock_listdir): self.mock_instance_config.GetOptionString.return_value = None @@ -325,6 +364,7 @@ mock_instance_id.return_value = '123' self.mock_setup._GetInstanceId = mock_instance_id mock_generate_key = mock.Mock() + mock_generate_key.return_value = ('ssh-rsa', 'asdfasdf') self.mock_setup._GenerateSshKey = mock_generate_key mock_listdir.return_value = [ 'ssh_config', @@ -344,6 +384,7 @@ mock.call('ed25519', '/etc/ssh/ssh_host_ed25519_key'), mock.call('rsa', '/etc/ssh/ssh_host_rsa_key'), ] + self.assertEqual(sorted(mock_generate_key.mock_calls), expected_calls) self.mock_instance_config.SetOption.assert_called_once_with( 'Instance', 'instance_id', '123') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py --- old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py 2019-05-22 22:41:34.000000000 +0200 @@ -15,11 +15,11 @@ """Retrieve and store user provided metadata scripts.""" -import ast +import functools import re import socket -import subprocess import tempfile +import time from google_compute_engine import metadata_watcher from google_compute_engine.compat import httpclient @@ -28,6 +28,37 @@ from google_compute_engine.compat import urlretrieve +def _RetryOnUnavailable(func): + """Function decorator template to retry on a service unavailable exception.""" + + @functools.wraps(func) + def Wrapper(*args, **kwargs): + final_exception = None + for _ in range(3): + try: + response = func(*args, **kwargs) + except (httpclient.HTTPException, socket.error, urlerror.URLError) as e: + final_exception = e + time.sleep(5) + continue + else: + return response + raise final_exception + return Wrapper + + +@_RetryOnUnavailable +def _UrlOpenWithRetry(request): + """Call urlopen with retry.""" + return urlrequest.urlopen(request) + + +@_RetryOnUnavailable +def _UrlRetrieveWithRetry(url, dest): + """Call urlretrieve with retry.""" + return urlretrieve.urlretrieve(url, dest) + + class ScriptRetriever(object): """A class for retrieving and storing user provided metadata scripts.""" token_metadata_key = 'instance/service-accounts/default/token' @@ -81,12 +112,12 @@ request = urlrequest.Request(url) request.add_unredirected_header('Metadata-Flavor', 'Google') request.add_unredirected_header('Authorization', self.token) - content = urlrequest.urlopen(request).read().decode('utf-8') - except (httpclient.HTTPException, socket.error, urlerror.URLError) as e: + content = _UrlOpenWithRetry(request).read().decode('utf-8') + except Exception as e: self.logger.warning('Could not download %s. %s.', url, str(e)) return None - with open(dest, 'wb') as f: + with open(dest, 'w') as f: f.write(content) return dest @@ -107,7 +138,7 @@ self.logger.info('Downloading url from %s to %s.', url, dest) try: - urlretrieve.urlretrieve(url, dest) + _UrlRetrieveWithRetry(url, dest) return dest except (httpclient.HTTPException, socket.error, urlerror.URLError) as e: self.logger.warning('Could not download %s. %s.', url, str(e)) @@ -192,8 +223,10 @@ metadata_value = attribute_data.get(metadata_key) if metadata_value: self.logger.info('Found %s in metadata.', metadata_key) - script_dict[metadata_key] = self._DownloadScript( - metadata_value, dest_dir) + downloaded_dest = self._DownloadScript(metadata_value, dest_dir) + if downloaded_dest is None: + self.logger.warning('Failed to download metadata script.') + script_dict[metadata_key] = downloaded_dest return script_dict diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py --- old/compute-image-packages-20190416/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py 2019-05-22 22:41:34.000000000 +0200 @@ -19,7 +19,6 @@ from google_compute_engine.compat import urlerror from google_compute_engine.metadata_scripts import script_retriever -from google_compute_engine.metadata_watcher import MetadataWatcher from google_compute_engine.test_compat import builtin from google_compute_engine.test_compat import mock from google_compute_engine.test_compat import unittest @@ -63,7 +62,7 @@ urlopen_read = mock_urlopen().read(return_value=b'foo').decode() self.mock_logger.warning.assert_not_called() - mock_open.assert_called_once_with(self.dest, 'wb') + mock_open.assert_called_once_with(self.dest, 'w') handle = mock_open() handle.write.assert_called_once_with(urlopen_read) @@ -145,16 +144,42 @@ mock_retrieve.assert_called_once_with(url, self.dest) self.mock_logger.warning.assert_not_called() + @mock.patch('google_compute_engine.metadata_scripts.script_retriever.time') @mock.patch('google_compute_engine.metadata_scripts.script_retriever.tempfile.NamedTemporaryFile') @mock.patch('google_compute_engine.metadata_scripts.script_retriever.urlretrieve.urlretrieve') - def testDownloadUrlProcessError(self, mock_retrieve, mock_tempfile): + def testDownloadUrlProcessError(self, mock_retrieve, mock_tempfile, mock_time): url = 'http://www.google.com/fake/url' mock_tempfile.return_value = mock_tempfile mock_tempfile.name = self.dest - mock_retrieve.side_effect = script_retriever.socket.timeout() + mock_success = mock.Mock() + mock_success.getcode.return_value = script_retriever.httpclient.OK + # Success after 3 timeout. Since max_retry = 3, the final result is fail. + mock_retrieve.side_effect = [ + script_retriever.socket.timeout(), + script_retriever.socket.timeout(), + script_retriever.socket.timeout(), + mock_success, + ] self.assertIsNone(self.retriever._DownloadUrl(url, self.dest_dir)) self.assertEqual(self.mock_logger.warning.call_count, 1) + @mock.patch('google_compute_engine.metadata_scripts.script_retriever.time') + @mock.patch('google_compute_engine.metadata_scripts.script_retriever.tempfile.NamedTemporaryFile') + @mock.patch('google_compute_engine.metadata_scripts.script_retriever.urlretrieve.urlretrieve') + def testDownloadUrlWithRetry(self, mock_retrieve, mock_tempfile, mock_time): + url = 'http://www.google.com/fake/url' + mock_tempfile.return_value = mock_tempfile + mock_tempfile.name = self.dest + mock_success = mock.Mock() + mock_success.getcode.return_value = script_retriever.httpclient.OK + # Success after 2 timeout. Since max_retry = 3, the final result is success. + mock_retrieve.side_effect = [ + script_retriever.socket.timeout(), + script_retriever.socket.timeout(), + mock_success, + ] + self.assertIsNotNone(self.retriever._DownloadUrl(url, self.dest_dir)) + @mock.patch('google_compute_engine.metadata_scripts.script_retriever.tempfile.NamedTemporaryFile') @mock.patch('google_compute_engine.metadata_scripts.script_retriever.urlretrieve.urlretrieve') def testDownloadUrlException(self, mock_retrieve, mock_tempfile): @@ -325,6 +350,7 @@ self.assertEqual(self.retriever.GetScripts(self.dest_dir), expected_data) self.assertEqual(self.mock_logger.info.call_count, 2) + self.assertEqual(self.mock_logger.warning.call_count, 0) mock_dest.write.assert_called_once_with('a') mock_download.assert_called_once_with('b', self.dest_dir) @@ -352,6 +378,44 @@ self.mock_logger.info.assert_not_called() self.assertEqual(self.mock_logger.warning.call_count, 2) + @mock.patch('google_compute_engine.metadata_scripts.script_retriever.tempfile.NamedTemporaryFile') + def testGetScriptsFailed(self, mock_tempfile): + script_dest = '/tmp/script' + script_url_dest = None + metadata = { + 'instance': { + 'attributes': { + '%s-script' % self.script_type: 'a', + '%s-script-url' % self.script_type: 'b', + }, + }, + 'project': { + 'attributes': { + '%s-script' % self.script_type: 'c', + '%s-script-url' % self.script_type: 'd', + }, + }, + } + expected_data = { + '%s-script' % self.script_type: script_dest, + '%s-script-url' % self.script_type: script_url_dest, + } + self.mock_watcher.GetMetadata.return_value = metadata + self.retriever.watcher = self.mock_watcher + # Mock saving a script to a file. + mock_dest = mock.Mock() + mock_dest.name = script_dest + mock_tempfile.__enter__.return_value = mock_dest + mock_tempfile.return_value = mock_tempfile + # Mock downloading a script from a URL. + mock_download = mock.Mock() + mock_download.return_value = None + self.retriever._DownloadScript = mock_download + + self.assertEqual(self.retriever.GetScripts(self.dest_dir), expected_data) + self.assertEqual(self.mock_logger.info.call_count, 2) + self.assertEqual(self.mock_logger.warning.call_count, 1) + if __name__ == '__main__': unittest.main() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/packaging/debian/changelog new/compute-image-packages-20190522/packages/python-google-compute-engine/packaging/debian/changelog --- old/compute-image-packages-20190416/packages/python-google-compute-engine/packaging/debian/changelog 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/packaging/debian/changelog 2019-05-22 22:41:34.000000000 +0200 @@ -1,3 +1,18 @@ +python-google-compute-engine (2.8.16-1) stable; urgency=low + + * Fix guest attributes flow in Python 3. + + -- Google Cloud Team Tue, 22 May 2019 12:00:00 -0700 + +python-google-compute-engine (2.8.15-1) stable; urgency=low + + * Retry download for metadata scripts. + * Fix script retrieval in python3. + * Disable boto config in python3. + * Update SSH host keys in guest attributes. + + -- Google Cloud Team Tue, 21 May 2019 12:00:00 -0700 + python-google-compute-engine (2.8.14-1) stable; urgency=low * FreeBSD fixes: syslog socket location and OS detection. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/packaging/python3-google-compute-engine.spec new/compute-image-packages-20190522/packages/python-google-compute-engine/packaging/python3-google-compute-engine.spec --- old/compute-image-packages-20190416/packages/python-google-compute-engine/packaging/python3-google-compute-engine.spec 1970-01-01 01:00:00.000000000 +0100 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/packaging/python3-google-compute-engine.spec 2019-05-22 22:41:34.000000000 +0200 @@ -0,0 +1,47 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Name: python3-google-compute-engine +Version: %{_version} +Release: 1%{?dist} +Summary: Google Compute Engine python3 library +License: ASL 2.0 +Url: https://github.com/GoogleCloudPlatform/compute-image-packages +Source0: %{name}_%{version}.orig.tar.gz + +BuildArch: noarch +BuildRequires: python36-devel python3-setuptools + +Requires: python3-setuptools + +%description +Google Compute Engine python library for Python 3.x. + +%prep +%autosetup + +%build +%py3_build + +%install +%py3_install + +%files +%{python3_sitelib}/google_compute_engine/ +%{python3_sitelib}/google_compute_engine*.egg-info/ +%{_bindir}/google_accounts_daemon +%{_bindir}/google_clock_skew_daemon +%{_bindir}/google_instance_setup +%{_bindir}/google_metadata_script_runner +%{_bindir}/google_network_daemon diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/packaging/setup_deb.sh new/compute-image-packages-20190522/packages/python-google-compute-engine/packaging/setup_deb.sh --- old/compute-image-packages-20190416/packages/python-google-compute-engine/packaging/setup_deb.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/packaging/setup_deb.sh 2019-05-22 22:41:34.000000000 +0200 @@ -14,7 +14,7 @@ # limitations under the License. NAME="python-google-compute-engine" -VERSION="2.8.14" +VERSION="2.8.16" working_dir=${PWD} if [[ $(basename "$working_dir") != $NAME ]]; then diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/packaging/setup_rpm.sh new/compute-image-packages-20190522/packages/python-google-compute-engine/packaging/setup_rpm.sh --- old/compute-image-packages-20190416/packages/python-google-compute-engine/packaging/setup_rpm.sh 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/packaging/setup_rpm.sh 2019-05-22 22:41:34.000000000 +0200 @@ -14,7 +14,7 @@ # limitations under the License. NAME="python-google-compute-engine" -VERSION="2.8.14" +VERSION="2.8.16" rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION} working_dir=${PWD} @@ -23,12 +23,17 @@ exit 1 fi -# Build dependencies. -sudo yum -y install python2-devel python-setuptools python-boto - -# RPM creation tools. sudo yum -y install rpmdevtools +# RHEL/CentOS 8 uses python3. +if grep -q '^\(CentOS\|Red Hat\)[^0-9]*8\..' /etc/redhat-release; then + NAME="python3-google-compute-engine" + rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION} + sudo yum -y install python36-devel python3-setuptools python36-rpm-macros +else + sudo yum -y install python2-devel python-setuptools python-boto +fi + rm -rf /tmp/rpmpackage mkdir -p ${rpm_working_dir}/{SOURCES,SPECS} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/compute-image-packages-20190416/packages/python-google-compute-engine/setup.py new/compute-image-packages-20190522/packages/python-google-compute-engine/setup.py --- old/compute-image-packages-20190416/packages/python-google-compute-engine/setup.py 2019-04-17 02:46:12.000000000 +0200 +++ new/compute-image-packages-20190522/packages/python-google-compute-engine/setup.py 2019-05-22 22:41:34.000000000 +0200 @@ -20,8 +20,10 @@ import setuptools -install_requires = ['boto', 'setuptools'] -if sys.version_info >= (3, 6): +install_requires = ['setuptools'] +if sys.version_info < (3, 0): + install_requires += ['boto'] +if sys.version_info >= (3, 7): install_requires += ['distro'] setuptools.setup( @@ -35,7 +37,7 @@ name='google-compute-engine', packages=setuptools.find_packages(), url='https://github.com/GoogleCloudPlatform/compute-image-packages', - version='2.8.13', + version='2.8.16', # Entry points create scripts in /usr/bin that call a function. entry_points={ 'console_scripts': [