Merge branch 'development'
diff --git a/README.md b/README.md
index f7b698b..ed306f3 100644
--- a/README.md
+++ b/README.md
@@ -43,10 +43,11 @@
CentOS 7 | rpm | 2.7 | systemd
RHEL 6 | rpm | 2.6 | upstart
RHEL 7 | rpm | 2.7 | systemd
+RHEL 8 | rpm | 3.6 | systemd
Ubuntu 14.04 | deb | 2.7 | upstart
Ubuntu 16.04 | deb | 3.5 or 2.7 | systemd
Ubuntu 18.04 | deb | 3.6 | systemd
-Ubuntu 18.10 | deb | 3.6 | systemd
+Ubuntu 19.04 | deb | 3.7 | systemd
Debian 9 | deb | 3.5 or 2.7 | systemd
We build the following packages for the Linux guest environment.
@@ -123,9 +124,9 @@
```
DIST=7
tee /etc/yum.repos.d/google-cloud.repo << EOM
-[google-cloud-compute]
-name=Google Cloud Compute
-baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el${DIST}-x86_64
+[google-compute-engine]
+name=Google Compute Engine
+baseurl=https://packages.cloud.google.com/yum/repos/google-compute-engine-el${DIST}-x86_64-stable
enabled=1
gpgcheck=1
repo_gpgcheck=1
diff --git a/daisy_workflows/build_el.wf.json b/daisy_workflows/build_el.wf.json
index 1f577f9..13303fd 100644
--- a/daisy_workflows/build_el.wf.json
+++ b/daisy_workflows/build_el.wf.json
@@ -21,6 +21,12 @@
"setup-disk": {
"CreateDisks": [
{
+ "Name": "disk-el8-build",
+ "SourceImage": "projects/rhel-cloud/global/images/family/rhel-8",
+ "SizeGb": "10",
+ "Type": "pd-ssd"
+ },
+ {
"Name": "disk-el7-build",
"SourceImage": "projects/centos-cloud/global/images/family/centos-7",
"SizeGb": "10",
@@ -37,6 +43,20 @@
"package-build": {
"CreateInstances": [
{
+ "Name": "inst-el8-build",
+ "Disks": [
+ {"Source": "disk-el8-build"}
+ ],
+ "MachineType": "n1-standard-2",
+ "Metadata": {
+ "github_branch": "${github_branch}",
+ "github_repo": "${github_repo}",
+ "output_path": "${output_path}"
+ },
+ "Scopes": ["https://www.googleapis.com/auth/devstorage.read_write"],
+ "StartupScript": "build_el_packages.sh"
+ },
+ {
"Name": "inst-el7-build",
"Disks": [
{"Source": "disk-el7-build"}
@@ -69,6 +89,14 @@
"wait-for-build": {
"WaitForInstancesSignal": [
{
+ "Name": "inst-el8-build",
+ "SerialOutput": {
+ "Port": 1,
+ "SuccessMatch": "BuildSuccess:",
+ "FailureMatch": "BuildFailed:"
+ }
+ },
+ {
"Name": "inst-el7-build",
"SerialOutput": {
"Port": 1,
diff --git a/packages/gce-disk-expand/packaging/gce-disk-expand.spec b/packages/gce-disk-expand/packaging/gce-disk-expand.spec
index d34b08d..a1fb6dd 100644
--- a/packages/gce-disk-expand/packaging/gce-disk-expand.spec
+++ b/packages/gce-disk-expand/packaging/gce-disk-expand.spec
@@ -33,13 +33,13 @@
%install
mv src/expandfs-lib.sh src/usr/share/dracut/modules.d/50expand_rootfs/
-%if 0%{?rhel} == 7
+%if 0%{?rhel} >= 7
./dracut6_7.sh
%endif
rsync -Pravz src/ %{buildroot}
%files
-%if 0%{?rhel} == 7
+%if 0%{?rhel} >= 7
%attr(755,root,root) /usr/lib/dracut/modules.d/50expand_rootfs/*
%else
%attr(755,root,root) /usr/share/dracut/modules.d/50expand_rootfs/*
diff --git a/packages/gce-disk-expand/packaging/setup_deb.sh b/packages/gce-disk-expand/packaging/setup_deb.sh
index 8880efe..ca5be92 100755
--- a/packages/gce-disk-expand/packaging/setup_deb.sh
+++ b/packages/gce-disk-expand/packaging/setup_deb.sh
@@ -26,7 +26,7 @@
sudo apt-get -y install dh-systemd
# DEB creation tools.
-sudo apt-get -y install debhelper devscripts build-essential
+sudo apt-get -y install debhelper devscripts build-essential rsync
rm -rf /tmp/debpackage
mkdir /tmp/debpackage
diff --git a/packages/gce-disk-expand/packaging/setup_rpm.sh b/packages/gce-disk-expand/packaging/setup_rpm.sh
index 52f6a36..106aa6f 100755
--- a/packages/gce-disk-expand/packaging/setup_rpm.sh
+++ b/packages/gce-disk-expand/packaging/setup_rpm.sh
@@ -24,7 +24,7 @@
fi
# RPM creation tools.
-sudo yum -y install rpmdevtools
+sudo yum -y install rpmdevtools rsync
rm -rf ${rpm_working_dir}
mkdir -p ${rpm_working_dir}/{SOURCES,SPECS}
diff --git a/packages/gce-disk-expand/src/expandfs-lib.sh b/packages/gce-disk-expand/src/expandfs-lib.sh
index 5126602..92e9ed5 100755
--- a/packages/gce-disk-expand/src/expandfs-lib.sh
+++ b/packages/gce-disk-expand/src/expandfs-lib.sh
@@ -24,7 +24,7 @@
case "${fs_type}" in
xfs)
echo "XFS filesystems must be mounted to be resized, deferring."
- expand_xfs="true"
+ echo "true" > /tmp/xfs_resize
return 1
;;
ext*)
@@ -136,6 +136,7 @@
echo "Unable to resize ${disk}${partnum}: ${out}"
return 1
fi
+ udevadm settle
}
# Resizes partition by deleting and recreating with end position.
@@ -184,4 +185,5 @@
return 1
fi
done
+ udevadm settle
)
diff --git a/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs.sh b/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs.sh
index 055f083..cc831b3 100755
--- a/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs.sh
+++ b/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs.sh
@@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-expand_xfs="" # Global to message next script.
-
# Contains dracut-specific logic for detecting disk, then calls appropriate
# library functions.
main() {
@@ -26,11 +24,7 @@
return
fi
- # Wait for any of the initial udev events to finish otherwise growpart
- # might fail.
- udevsettle
-
- if ! out=$(get_partition "$rootdev"); then
+ if ! out=$(split_partition "$rootdev"); then
echo "Failed to detect disk and partition info: ${out}"
return
fi
@@ -63,8 +57,6 @@
fi
fi
- udevsettle
-
if ! out=$(resize_filesystem "$rootdev"); then
echo "Failed to resize filesystem: ${out}"
return
diff --git a/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs_dummy.sh b/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs_dummy.sh
new file mode 100755
index 0000000..c037109
--- /dev/null
+++ b/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/expand_rootfs_dummy.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+# Dummy script, to make sure systemd executes the cmdline stage (which exports
+# the 'root' variable required for expand_root)
diff --git a/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/install b/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/install
index 4b68069..cdd021d 100755
--- a/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/install
+++ b/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/install
@@ -16,8 +16,9 @@
# Dracut install script for RHEL/CentOS
inst "$moddir/expandfs-lib.sh" "/lib/expandfs-lib.sh"
+inst_hook cmdline 50 "$moddir/expand_rootfs_dummy.sh"
inst_hook pre-mount 50 "$moddir/expand_rootfs.sh"
-inst_hook pre-pivot 50 "$moddir/xfs_growfs.sh"
+inst_hook pre-pivot 99 "$moddir/xfs_growfs.sh"
dracut_install parted
dracut_install cut
@@ -25,4 +26,5 @@
dracut_install grep
dracut_install resize2fs
dracut_install e2fsck
+dracut_install udevadm
dracut_install -o xfs_growfs
diff --git a/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/xfs_growfs.sh b/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/xfs_growfs.sh
index af8d05d..0e77185 100755
--- a/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/xfs_growfs.sh
+++ b/packages/gce-disk-expand/src/usr/share/dracut/modules.d/50expand_rootfs/xfs_growfs.sh
@@ -14,7 +14,7 @@
# limitations under the License.
main() {
- if [ -z "$expand_xfs" ]; then
+ if [ ! -e /tmp/xfs_resize ]; then
return
fi
@@ -23,11 +23,18 @@
return
fi
if xfs_growfs -d -n /sysroot; then
+ echo "Mounting filesystem rw."
+ if ! $(mount -o rw,remount /sysroot); then
+ echo "Remount failed."
+ return
+ fi
echo "Resizing XFS filesystem"
if ! out=$(xfs_growfs -d /sysroot); then
echo "Failed to resize: ${out}"
+ mount -o ro,remount /sysroot
return
fi
+ mount -o ro,remount /sysroot
fi
}
diff --git a/packages/google-compute-engine-oslogin/packaging/google-compute-engine-oslogin.spec b/packages/google-compute-engine-oslogin/packaging/google-compute-engine-oslogin.spec
index 7e0799d..b8b0e67 100644
--- a/packages/google-compute-engine-oslogin/packaging/google-compute-engine-oslogin.spec
+++ b/packages/google-compute-engine-oslogin/packaging/google-compute-engine-oslogin.spec
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Force the dist to be el7 to avoid el7.centos.
+# For EL7, if building on CentOS, override dist to be el7.
%if 0%{?rhel} == 7
%define dist .el7
%endif
@@ -29,11 +29,17 @@
BuildRequires: gcc-c++
BuildRequires: make
BuildRequires: libcurl
-BuildRequires: json-c
+BuildRequires: json-c-devel
BuildRequires: pam-devel
+%if 0%{?rhel} == 8
+BuildRequires: python3-policycoreutils
+Requires: python3-policycoreutils
+%else
BuildRequires: policycoreutils-python
-Requires: boost-regex
Requires: policycoreutils-python
+%endif
+Requires: boost-regex
+Requires: json-c
%define pam_install_path /%{_lib}/security
@@ -41,6 +47,8 @@
This package contains several libraries and changes to enable OS Login functionality
for Google Compute Engine.
+%global debug_package %{nil}
+
%prep
%setup
@@ -53,10 +61,14 @@
%files
%doc
-/%{_lib}/libnss_%{name}-%{version}.so
-/%{_lib}/libnss_cache_%{name}-%{version}.so
-%{pam_install_path}/pam_oslogin_admin.so
-%{pam_install_path}/pam_oslogin_login.so
+%attr(0755,-,-) /%{_lib}/libnss_%{name}-%{version}.so
+%attr(0755,-,-) /%{_lib}/libnss_cache_%{name}-%{version}.so
+%if 0%{?rhel} == 8
+/%{_lib}/libnss_oslogin.so.2
+/%{_lib}/libnss_cache_oslogin.so.2
+%endif
+%attr(0755,-,-) %{pam_install_path}/pam_oslogin_admin.so
+%attr(0755,-,-) %{pam_install_path}/pam_oslogin_login.so
/usr/bin/google_authorized_keys
/usr/bin/google_oslogin_control
/usr/bin/google_oslogin_nss_cache
diff --git a/packages/google-compute-engine-oslogin/packaging/setup_rpm.sh b/packages/google-compute-engine-oslogin/packaging/setup_rpm.sh
index f174efe..519247d 100755
--- a/packages/google-compute-engine-oslogin/packaging/setup_rpm.sh
+++ b/packages/google-compute-engine-oslogin/packaging/setup_rpm.sh
@@ -24,11 +24,14 @@
exit 1
fi
-# Build dependencies.
-sudo yum -y install make gcc-c++ libcurl-devel json-c json-c-devel pam-devel policycoreutils-python boost-devel
+sudo yum -y install rpmdevtools make gcc-c++ json-c \
+ libcurl-devel pam-devel boost-devel json-c-devel
-# RPM creation tools.
-sudo yum -y install rpmdevtools
+if grep -q '^\(CentOS\|Red Hat\)[^0-9]*8\..' /etc/redhat-release; then
+ sudo yum -y install python3-policycoreutils
+else
+ sudo yum -y install policycoreutils-python
+fi
rm -rf /tmp/rpmpackage
mkdir -p ${rpm_working_dir}/{SOURCES,SPECS}
diff --git a/packages/google-compute-engine/packaging/debian/changelog b/packages/google-compute-engine/packaging/debian/changelog
index a9e9396..862f51e 100644
--- a/packages/google-compute-engine/packaging/debian/changelog
+++ b/packages/google-compute-engine/packaging/debian/changelog
@@ -1,3 +1,9 @@
+google-compute-engine (2.8.15-1) stable; urgency=low
+
+ * Fix XPS settings with more than 64 vCPUs.
+
+ -- Google Cloud Team <gc-team@google.com> Tue, 21 May 2019 12:00:00 -0700
+
google-compute-engine (2.8.14-1) stable; urgency=low
* Upstart systems: only run startup scripts at boot.
diff --git a/packages/google-compute-engine/packaging/google-compute-engine.spec b/packages/google-compute-engine/packaging/google-compute-engine.spec
index 39307cb..aba148d 100644
--- a/packages/google-compute-engine/packaging/google-compute-engine.spec
+++ b/packages/google-compute-engine/packaging/google-compute-engine.spec
@@ -12,16 +12,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# For EL7, if building on CentOS, override dist to be el7.
+%if 0%{?rhel} == 7
+ %define dist .el7
+%endif
+
Name: google-compute-engine
Version: %{_version}
-Release: 1.el7
+Release: 1%{?dist}
Summary: Google Compute Engine guest environment.
License: ASL 2.0
Url: https://github.com/GoogleCloudPlatform/compute-image-packages
Source0: %{name}_%{version}.orig.tar.gz
Requires: curl
Requires: google-compute-engine-oslogin
+%if 0%{?rhel} == 8
+Requires: python3-google-compute-engine = %{version}
+%else
Requires: python-google-compute-engine = %{version}
+%endif
Requires: rsyslog
BuildArch: noarch
diff --git a/packages/google-compute-engine/packaging/setup_deb.sh b/packages/google-compute-engine/packaging/setup_deb.sh
index 76cce07..7ad1b02 100755
--- a/packages/google-compute-engine/packaging/setup_deb.sh
+++ b/packages/google-compute-engine/packaging/setup_deb.sh
@@ -14,7 +14,7 @@
# limitations under the License.
NAME="google-compute-engine"
-VERSION="2.8.14"
+VERSION="2.8.15"
working_dir=${PWD}
if [[ $(basename "$working_dir") != $NAME ]]; then
diff --git a/packages/google-compute-engine/packaging/setup_rpm.sh b/packages/google-compute-engine/packaging/setup_rpm.sh
index d98c2aa..8f5dfa9 100755
--- a/packages/google-compute-engine/packaging/setup_rpm.sh
+++ b/packages/google-compute-engine/packaging/setup_rpm.sh
@@ -14,7 +14,7 @@
# limitations under the License.
NAME="google-compute-engine"
-VERSION="2.8.14"
+VERSION="2.8.15"
rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION}
working_dir=${PWD}
@@ -23,10 +23,6 @@
exit 1
fi
-# Build dependencies.
-sudo yum -y install make gcc-c++ libcurl-devel json-c json-c-devel pam-devel \
- policycoreutils-python boost-devel
-
# RPM creation tools.
sudo yum -y install rpmdevtools
diff --git a/packages/google-compute-engine/src/lib/systemd/system/google-accounts-daemon.service b/packages/google-compute-engine/src/lib/systemd/system/google-accounts-daemon.service
index 8ea5573..9158be5 100644
--- a/packages/google-compute-engine/src/lib/systemd/system/google-accounts-daemon.service
+++ b/packages/google-compute-engine/src/lib/systemd/system/google-accounts-daemon.service
@@ -7,6 +7,7 @@
Type=simple
ExecStart=/usr/bin/google_accounts_daemon
OOMScoreAdjust=-999
+Restart=always
[Install]
WantedBy=multi-user.target
diff --git a/packages/google-compute-engine/src/usr/bin/google_set_multiqueue b/packages/google-compute-engine/src/usr/bin/google_set_multiqueue
index 9dfe45f..7064a6b 100755
--- a/packages/google-compute-engine/src/usr/bin/google_set_multiqueue
+++ b/packages/google-compute-engine/src/usr/bin/google_set_multiqueue
@@ -125,10 +125,16 @@
xps=$((xps | (1 << cpu)))
done
- # Linux xps_cpus requires a hex number with commas every 32 bits.
- # It ignores all bits above # cpus, so unconditionally write a
- # 64 bit hex value, with a comma between dwords.
- xps_string=`printf "%08x,%08x" $((xps >> 32 & 0xffffffff)) $((xps & 0xffffffff))`
+ # Linux xps_cpus requires a hex number with commas every 32 bits. It ignores
+ # all bits above # cpus, so write a list of comma separated 32 bit hex values
+ # with a comma between dwords.
+ xps_dwords=()
+ for i in $(seq 0 $(((num_cpus - 1) / 32)))
+ do
+ xps_dwords+=(`printf "%08x" $((xps & 0xffffffff))`)
+ done
+ xps_string=$(IFS=, ; echo "${xps_dwords[*]}")
+
echo ${xps_string} > $q
printf "Queue %d XPS=%s for %s\n" $queue_num `cat $q` $q
diff --git a/packages/python-google-compute-engine/README.md b/packages/python-google-compute-engine/README.md
index f1ffe8e..0bd6ddb 100644
--- a/packages/python-google-compute-engine/README.md
+++ b/packages/python-google-compute-engine/README.md
@@ -28,8 +28,6 @@
based authentication.
* **Clock skew** daemon to keep the system clock in sync after VM start and
stop events.
-* **Disk expand** scripts to expand the VM root partition for CentOS 6,
- CentOS 7, RHEL 6, and RHEL 7 images.
* **Instance setup** scripts to execute VM configuration scripts during boot.
* **Network** daemon that handles network setup for multiple network interfaces
on boot and integrates network load balancing with
@@ -38,7 +36,7 @@
shutdown.
The Linux guest environment is written in Python and is version agnostic
-between Python 2.6 and 3.5. There is complete unittest coverage for every Python
+between Python 2.6 and 3.7. There is complete unittest coverage for every Python
library and script. The design of various guest libraries, daemons, and scripts,
are detailed in the sections below.
diff --git a/packages/python-google-compute-engine/google_compute_engine/boto/tests/compute_auth_test.py b/packages/python-google-compute-engine/google_compute_engine/boto/tests/compute_auth_test.py
index 0c31052..93e55ae 100644
--- a/packages/python-google-compute-engine/google_compute_engine/boto/tests/compute_auth_test.py
+++ b/packages/python-google-compute-engine/google_compute_engine/boto/tests/compute_auth_test.py
@@ -15,11 +15,16 @@
"""Unittest for compute_auth.py module."""
-from google_compute_engine.boto import compute_auth
+import sys
+
from google_compute_engine.test_compat import mock
from google_compute_engine.test_compat import unittest
+if sys.version_info < (3, 0):
+ from google_compute_engine.boto import compute_auth
+
+@unittest.skipIf(sys.version_info > (3, 0), 'Skipping for python3.')
class ComputeAuthTest(unittest.TestCase):
def setUp(self):
diff --git a/packages/python-google-compute-engine/google_compute_engine/compat.py b/packages/python-google-compute-engine/google_compute_engine/compat.py
index eb69763..3d18c12 100644
--- a/packages/python-google-compute-engine/google_compute_engine/compat.py
+++ b/packages/python-google-compute-engine/google_compute_engine/compat.py
@@ -19,7 +19,7 @@
import subprocess
import sys
-if sys.version_info >= (3, 6):
+if sys.version_info >= (3, 7):
import distro
else:
import platform as distro
diff --git a/packages/python-google-compute-engine/google_compute_engine/constants.py b/packages/python-google-compute-engine/google_compute_engine/constants.py
index cfe87c4..d929baf 100644
--- a/packages/python-google-compute-engine/google_compute_engine/constants.py
+++ b/packages/python-google-compute-engine/google_compute_engine/constants.py
@@ -16,6 +16,7 @@
"""A module for global constants."""
import platform
+import sys
OSLOGIN_CONTROL_SCRIPT = 'google_oslogin_control'
OSLOGIN_NSS_CACHE_SCRIPT = 'google_oslogin_nss_cache'
@@ -41,3 +42,8 @@
OSLOGIN_NSS_CACHE = '/etc/oslogin_passwd.cache'
SYSCONFDIR = '/etc/default'
SYSLOG_SOCKET = '/dev/log'
+
+if sys.version_info >= (3, 0):
+ SET_BOTO_CONFIG = 'false'
+else:
+ SET_BOTO_CONFIG = 'true'
diff --git a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py
index 1c23171..2326115 100644
--- a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py
+++ b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py
@@ -78,7 +78,7 @@
'host_key_types': 'ecdsa,ed25519,rsa',
'optimize_local_ssd': 'true',
'network_enabled': 'true',
- 'set_boto_config': 'true',
+ 'set_boto_config': constants.SET_BOTO_CONFIG,
'set_host_keys': 'true',
'set_multiqueue': 'true',
},
diff --git a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py
index 80785e9..e72534c 100755
--- a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py
+++ b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py
@@ -28,9 +28,21 @@
from google_compute_engine import logger
from google_compute_engine import metadata_watcher
from google_compute_engine.boto import boto_config
+from google_compute_engine.compat import urlerror
+from google_compute_engine.compat import urlrequest
from google_compute_engine.instance_setup import instance_config
+class PutRequest(urlrequest.Request):
+ def get_method(self):
+ return 'PUT'
+
+
+GUEST_ATTRIBUTES_URL = ('http://metadata.google.internal/computeMetadata/v1beta1/'
+ 'instance/guest-attributes')
+HOSTKEY_NAMESPACE = 'hostkeys'
+
+
class InstanceSetup(object):
"""Initialize the instance the first time it boots."""
@@ -122,6 +134,9 @@
Args:
key_type: string, the type of the SSH key.
key_dest: string, a file location to store the SSH key.
+
+ Returns:
+ tuple, key_type and public key string.
"""
# Create a temporary file to save the created RSA keys.
with tempfile.NamedTemporaryFile(prefix=key_type, delete=True) as temp:
@@ -140,6 +155,28 @@
file_utils.SetPermissions(key_dest, mode=0o600)
file_utils.SetPermissions('%s.pub' % key_dest, mode=0o644)
+ with open('%s.pub' % key_dest, 'r') as pk:
+ key_data = pk.read()
+
+ key_values = key_data.split()
+ if len(key_values) < 2:
+ self.logger.warning('Could not read host key from %s.pub.', key_dest)
+ return
+ else:
+ return key_values[0], key_values[1]
+
+ def _WriteHostKeyToGuestAttributes(self, key_type, key_value):
+ """Write a host key to guest attributes, ignoring errors."""
+ headers = {'Metadata-Flavor': 'Google'}
+ url = '%s/%s/%s' % (GUEST_ATTRIBUTES_URL, HOSTKEY_NAMESPACE, key_type)
+ req = PutRequest(url, key_value, headers)
+ try:
+ response = urlrequest.urlopen(req)
+ self.logger.debug(response)
+ self.logger.info('Wrote %s host key to guest attributes.', key_type)
+ except urlerror.HTTPError:
+ self.logger.info('Unable to write %s host key to guest attributes.',
+ key_type)
def _StartSshd(self):
"""Initialize the SSH daemon."""
@@ -180,7 +217,9 @@
for key_file in set(key_files) | set(key_types_files):
key_type = file_regex.match(key_file).group('type')
key_dest = os.path.join(key_dir, key_file)
- self._GenerateSshKey(key_type, key_dest)
+ key_data = self._GenerateSshKey(key_type, key_dest)
+ if key_data:
+ self._WriteHostKeyToGuestAttributes(key_data[0], key_data[1])
self._StartSshd()
self.instance_config.SetOption(section, 'instance_id', str(instance_id))
diff --git a/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py b/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py
index 97a4674..a59ae0c 100644
--- a/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py
+++ b/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py
@@ -18,6 +18,7 @@
import subprocess
from google_compute_engine.instance_setup import instance_setup
+from google_compute_engine.test_compat import builtin
from google_compute_engine.test_compat import mock
from google_compute_engine.test_compat import unittest
@@ -231,22 +232,33 @@
temp_dest = '/tmp/dest'
mock_tempfile.return_value = mock_tempfile
mock_tempfile.__enter__.return_value.name = temp_dest
+ mock_open = mock.mock_open()
+ key_file_contents = 'ssh-rsa asdfasdf'
+ expected_key_data = ('ssh-rsa', 'asdfasdf')
- instance_setup.InstanceSetup._GenerateSshKey(
- self.mock_setup, key_type, key_dest)
- expected_calls = [
- mock.call.tempfile(prefix=key_type, delete=True),
- mock.call.tempfile.__enter__(),
- mock.call.tempfile.__exit__(None, None, None),
- mock.call.logger.info(mock.ANY, key_dest),
- mock.call.call(
- ['ssh-keygen', '-t', key_type, '-f', temp_dest, '-N', '', '-q']),
- mock.call.move(temp_dest, key_dest),
- mock.call.move('%s.pub' % temp_dest, '%s.pub' % key_dest),
- mock.call.permissions(key_dest, mode=0o600),
- mock.call.permissions('%s.pub' % key_dest, mode=0o644),
- ]
- self.assertEqual(mocks.mock_calls, expected_calls)
+ with mock.patch('%s.open' % builtin, mock_open, create=False):
+ mock_open().read.return_value = key_file_contents
+ key_data = instance_setup.InstanceSetup._GenerateSshKey(
+ self.mock_setup, key_type, key_dest)
+ expected_calls = [
+ mock.call.tempfile(prefix=key_type, delete=True),
+ mock.call.tempfile.__enter__(),
+ mock.call.tempfile.__exit__(None, None, None),
+ mock.call.logger.info(mock.ANY, key_dest),
+ mock.call.call(
+ ['ssh-keygen', '-t', key_type, '-f', temp_dest, '-N', '', '-q']),
+ mock.call.move(temp_dest, key_dest),
+ mock.call.move('%s.pub' % temp_dest, '%s.pub' % key_dest),
+ mock.call.permissions(key_dest, mode=0o600),
+ mock.call.permissions('%s.pub' % key_dest, mode=0o644),
+ ]
+ self.assertEqual(mocks.mock_calls, expected_calls)
+ self.assertEqual(key_data, expected_key_data)
+
+ mock_open().read.return_value = ''
+ key_data = instance_setup.InstanceSetup._GenerateSshKey(
+ self.mock_setup, key_type, key_dest)
+ self.assertEqual(key_data, None)
@mock.patch('google_compute_engine.instance_setup.instance_setup.subprocess.check_call')
def testGenerateSshKeyProcessError(self, mock_call):
@@ -318,6 +330,32 @@
instance_setup.InstanceSetup._SetSshHostKeys(self.mock_setup)
self.mock_instance_config.SetOption.assert_not_called()
+ @mock.patch('google_compute_engine.instance_setup.instance_setup.urlrequest.urlopen')
+ @mock.patch('google_compute_engine.instance_setup.instance_setup.PutRequest')
+ def testWriteHostKeyToGuestAttributes(self, mock_put, mock_urlopen):
+ key_type = 'ssh-rsa'
+ key_value = 'asdfasdf'
+ expected_url = ('http://metadata.google.internal/computeMetadata/v1beta1/'
+ 'instance/guest-attributes/hostkeys/%s' % key_type)
+ headers = {'Metadata-Flavor': 'Google'}
+
+ instance_setup.InstanceSetup._WriteHostKeyToGuestAttributes(
+ self.mock_setup, key_type, key_value)
+ self.mock_logger.info.assert_called_with(
+ 'Wrote %s host key to guest attributes.', key_type)
+ mock_put.assert_called_with(expected_url, key_value, headers)
+
+ mock_urlopen.side_effect = instance_setup.urlerror.HTTPError(
+ 'http://foo', 403, 'Forbidden', {}, None)
+ instance_setup.InstanceSetup._WriteHostKeyToGuestAttributes(
+ self.mock_setup, key_type, key_value)
+ self.mock_logger.info.assert_called_with(
+ 'Unable to write %s host key to guest attributes.', key_type)
+
+ def testPutRequest(self):
+ put_request = instance_setup.PutRequest('http://example.com/')
+ self.assertEqual(put_request.get_method(), 'PUT')
+
@mock.patch('google_compute_engine.instance_setup.instance_setup.os.listdir')
def testSetSshHostKeysFirstBoot(self, mock_listdir):
self.mock_instance_config.GetOptionString.return_value = None
@@ -325,6 +363,7 @@
mock_instance_id.return_value = '123'
self.mock_setup._GetInstanceId = mock_instance_id
mock_generate_key = mock.Mock()
+ mock_generate_key.return_value = ('ssh-rsa', 'asdfasdf')
self.mock_setup._GenerateSshKey = mock_generate_key
mock_listdir.return_value = [
'ssh_config',
@@ -344,6 +383,7 @@
mock.call('ed25519', '/etc/ssh/ssh_host_ed25519_key'),
mock.call('rsa', '/etc/ssh/ssh_host_rsa_key'),
]
+
self.assertEqual(sorted(mock_generate_key.mock_calls), expected_calls)
self.mock_instance_config.SetOption.assert_called_once_with(
'Instance', 'instance_id', '123')
diff --git a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py
index c678f99..c880f49 100644
--- a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py
+++ b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py
@@ -15,11 +15,11 @@
"""Retrieve and store user provided metadata scripts."""
-import ast
+import functools
import re
import socket
-import subprocess
import tempfile
+import time
from google_compute_engine import metadata_watcher
from google_compute_engine.compat import httpclient
@@ -28,6 +28,37 @@
from google_compute_engine.compat import urlretrieve
+def _RetryOnUnavailable(func):
+ """Function decorator template to retry on a service unavailable exception."""
+
+ @functools.wraps(func)
+ def Wrapper(*args, **kwargs):
+ final_exception = None
+ for _ in range(3):
+ try:
+ response = func(*args, **kwargs)
+ except (httpclient.HTTPException, socket.error, urlerror.URLError) as e:
+ final_exception = e
+ time.sleep(5)
+ continue
+ else:
+ return response
+ raise final_exception
+ return Wrapper
+
+
+@_RetryOnUnavailable
+def _UrlOpenWithRetry(request):
+ """Call urlopen with retry."""
+ return urlrequest.urlopen(request)
+
+
+@_RetryOnUnavailable
+def _UrlRetrieveWithRetry(url, dest):
+ """Call urlretrieve with retry."""
+ return urlretrieve.urlretrieve(url, dest)
+
+
class ScriptRetriever(object):
"""A class for retrieving and storing user provided metadata scripts."""
token_metadata_key = 'instance/service-accounts/default/token'
@@ -81,12 +112,12 @@
request = urlrequest.Request(url)
request.add_unredirected_header('Metadata-Flavor', 'Google')
request.add_unredirected_header('Authorization', self.token)
- content = urlrequest.urlopen(request).read().decode('utf-8')
- except (httpclient.HTTPException, socket.error, urlerror.URLError) as e:
+ content = _UrlOpenWithRetry(request).read().decode('utf-8')
+ except Exception as e:
self.logger.warning('Could not download %s. %s.', url, str(e))
return None
- with open(dest, 'wb') as f:
+ with open(dest, 'w') as f:
f.write(content)
return dest
@@ -107,7 +138,7 @@
self.logger.info('Downloading url from %s to %s.', url, dest)
try:
- urlretrieve.urlretrieve(url, dest)
+ _UrlRetrieveWithRetry(url, dest)
return dest
except (httpclient.HTTPException, socket.error, urlerror.URLError) as e:
self.logger.warning('Could not download %s. %s.', url, str(e))
@@ -192,8 +223,10 @@
metadata_value = attribute_data.get(metadata_key)
if metadata_value:
self.logger.info('Found %s in metadata.', metadata_key)
- script_dict[metadata_key] = self._DownloadScript(
- metadata_value, dest_dir)
+ downloaded_dest = self._DownloadScript(metadata_value, dest_dir)
+ if downloaded_dest is None:
+ self.logger.warning('Failed to download metadata script.')
+ script_dict[metadata_key] = downloaded_dest
return script_dict
diff --git a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py
index f3f520a..b9d9d1b 100644
--- a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py
+++ b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py
@@ -19,7 +19,6 @@
from google_compute_engine.compat import urlerror
from google_compute_engine.metadata_scripts import script_retriever
-from google_compute_engine.metadata_watcher import MetadataWatcher
from google_compute_engine.test_compat import builtin
from google_compute_engine.test_compat import mock
from google_compute_engine.test_compat import unittest
@@ -63,7 +62,7 @@
urlopen_read = mock_urlopen().read(return_value=b'foo').decode()
self.mock_logger.warning.assert_not_called()
- mock_open.assert_called_once_with(self.dest, 'wb')
+ mock_open.assert_called_once_with(self.dest, 'w')
handle = mock_open()
handle.write.assert_called_once_with(urlopen_read)
@@ -145,16 +144,42 @@
mock_retrieve.assert_called_once_with(url, self.dest)
self.mock_logger.warning.assert_not_called()
+ @mock.patch('google_compute_engine.metadata_scripts.script_retriever.time')
@mock.patch('google_compute_engine.metadata_scripts.script_retriever.tempfile.NamedTemporaryFile')
@mock.patch('google_compute_engine.metadata_scripts.script_retriever.urlretrieve.urlretrieve')
- def testDownloadUrlProcessError(self, mock_retrieve, mock_tempfile):
+ def testDownloadUrlProcessError(self, mock_retrieve, mock_tempfile, mock_time):
url = 'http://www.google.com/fake/url'
mock_tempfile.return_value = mock_tempfile
mock_tempfile.name = self.dest
- mock_retrieve.side_effect = script_retriever.socket.timeout()
+ mock_success = mock.Mock()
+ mock_success.getcode.return_value = script_retriever.httpclient.OK
+ # Success after 3 timeout. Since max_retry = 3, the final result is fail.
+ mock_retrieve.side_effect = [
+ script_retriever.socket.timeout(),
+ script_retriever.socket.timeout(),
+ script_retriever.socket.timeout(),
+ mock_success,
+ ]
self.assertIsNone(self.retriever._DownloadUrl(url, self.dest_dir))
self.assertEqual(self.mock_logger.warning.call_count, 1)
+ @mock.patch('google_compute_engine.metadata_scripts.script_retriever.time')
+ @mock.patch('google_compute_engine.metadata_scripts.script_retriever.tempfile.NamedTemporaryFile')
+ @mock.patch('google_compute_engine.metadata_scripts.script_retriever.urlretrieve.urlretrieve')
+ def testDownloadUrlWithRetry(self, mock_retrieve, mock_tempfile, mock_time):
+ url = 'http://www.google.com/fake/url'
+ mock_tempfile.return_value = mock_tempfile
+ mock_tempfile.name = self.dest
+ mock_success = mock.Mock()
+ mock_success.getcode.return_value = script_retriever.httpclient.OK
+ # Success after 2 timeout. Since max_retry = 3, the final result is success.
+ mock_retrieve.side_effect = [
+ script_retriever.socket.timeout(),
+ script_retriever.socket.timeout(),
+ mock_success,
+ ]
+ self.assertIsNotNone(self.retriever._DownloadUrl(url, self.dest_dir))
+
@mock.patch('google_compute_engine.metadata_scripts.script_retriever.tempfile.NamedTemporaryFile')
@mock.patch('google_compute_engine.metadata_scripts.script_retriever.urlretrieve.urlretrieve')
def testDownloadUrlException(self, mock_retrieve, mock_tempfile):
@@ -325,6 +350,7 @@
self.assertEqual(self.retriever.GetScripts(self.dest_dir), expected_data)
self.assertEqual(self.mock_logger.info.call_count, 2)
+ self.assertEqual(self.mock_logger.warning.call_count, 0)
mock_dest.write.assert_called_once_with('a')
mock_download.assert_called_once_with('b', self.dest_dir)
@@ -352,6 +378,44 @@
self.mock_logger.info.assert_not_called()
self.assertEqual(self.mock_logger.warning.call_count, 2)
+ @mock.patch('google_compute_engine.metadata_scripts.script_retriever.tempfile.NamedTemporaryFile')
+ def testGetScriptsFailed(self, mock_tempfile):
+ script_dest = '/tmp/script'
+ script_url_dest = None
+ metadata = {
+ 'instance': {
+ 'attributes': {
+ '%s-script' % self.script_type: 'a',
+ '%s-script-url' % self.script_type: 'b',
+ },
+ },
+ 'project': {
+ 'attributes': {
+ '%s-script' % self.script_type: 'c',
+ '%s-script-url' % self.script_type: 'd',
+ },
+ },
+ }
+ expected_data = {
+ '%s-script' % self.script_type: script_dest,
+ '%s-script-url' % self.script_type: script_url_dest,
+ }
+ self.mock_watcher.GetMetadata.return_value = metadata
+ self.retriever.watcher = self.mock_watcher
+ # Mock saving a script to a file.
+ mock_dest = mock.Mock()
+ mock_dest.name = script_dest
+ mock_tempfile.__enter__.return_value = mock_dest
+ mock_tempfile.return_value = mock_tempfile
+ # Mock downloading a script from a URL.
+ mock_download = mock.Mock()
+ mock_download.return_value = None
+ self.retriever._DownloadScript = mock_download
+
+ self.assertEqual(self.retriever.GetScripts(self.dest_dir), expected_data)
+ self.assertEqual(self.mock_logger.info.call_count, 2)
+ self.assertEqual(self.mock_logger.warning.call_count, 1)
+
if __name__ == '__main__':
unittest.main()
diff --git a/packages/python-google-compute-engine/packaging/debian/changelog b/packages/python-google-compute-engine/packaging/debian/changelog
index 822bcb9..1567ac1 100644
--- a/packages/python-google-compute-engine/packaging/debian/changelog
+++ b/packages/python-google-compute-engine/packaging/debian/changelog
@@ -1,3 +1,12 @@
+python-google-compute-engine (2.8.15-1) stable; urgency=low
+
+ * Retry download for metadata scripts.
+ * Fix script retrieval in python3.
+ * Disable boto config in python3.
+ * Update SSH host keys in guest attributes.
+
+ -- Google Cloud Team <gc-team@google.com> Tue, 21 May 2019 12:00:00 -0700
+
python-google-compute-engine (2.8.14-1) stable; urgency=low
* FreeBSD fixes: syslog socket location and OS detection.
diff --git a/packages/python-google-compute-engine/packaging/python3-google-compute-engine.spec b/packages/python-google-compute-engine/packaging/python3-google-compute-engine.spec
new file mode 100644
index 0000000..6bf3291
--- /dev/null
+++ b/packages/python-google-compute-engine/packaging/python3-google-compute-engine.spec
@@ -0,0 +1,47 @@
+# Copyright 2019 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Name: python3-google-compute-engine
+Version: %{_version}
+Release: 1%{?dist}
+Summary: Google Compute Engine python3 library
+License: ASL 2.0
+Url: https://github.com/GoogleCloudPlatform/compute-image-packages
+Source0: %{name}_%{version}.orig.tar.gz
+
+BuildArch: noarch
+BuildRequires: python36-devel python3-setuptools
+
+Requires: python3-setuptools
+
+%description
+Google Compute Engine python library for Python 3.x.
+
+%prep
+%autosetup
+
+%build
+%py3_build
+
+%install
+%py3_install
+
+%files
+%{python3_sitelib}/google_compute_engine/
+%{python3_sitelib}/google_compute_engine*.egg-info/
+%{_bindir}/google_accounts_daemon
+%{_bindir}/google_clock_skew_daemon
+%{_bindir}/google_instance_setup
+%{_bindir}/google_metadata_script_runner
+%{_bindir}/google_network_daemon
diff --git a/packages/python-google-compute-engine/packaging/setup_deb.sh b/packages/python-google-compute-engine/packaging/setup_deb.sh
index b47c54d..e3149a3 100755
--- a/packages/python-google-compute-engine/packaging/setup_deb.sh
+++ b/packages/python-google-compute-engine/packaging/setup_deb.sh
@@ -14,7 +14,7 @@
# limitations under the License.
NAME="python-google-compute-engine"
-VERSION="2.8.14"
+VERSION="2.8.15"
working_dir=${PWD}
if [[ $(basename "$working_dir") != $NAME ]]; then
diff --git a/packages/python-google-compute-engine/packaging/setup_rpm.sh b/packages/python-google-compute-engine/packaging/setup_rpm.sh
index d1eccc3..99c217f 100755
--- a/packages/python-google-compute-engine/packaging/setup_rpm.sh
+++ b/packages/python-google-compute-engine/packaging/setup_rpm.sh
@@ -14,7 +14,7 @@
# limitations under the License.
NAME="python-google-compute-engine"
-VERSION="2.8.14"
+VERSION="2.8.15"
rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION}
working_dir=${PWD}
@@ -23,12 +23,17 @@
exit 1
fi
-# Build dependencies.
-sudo yum -y install python2-devel python-setuptools python-boto
-
-# RPM creation tools.
sudo yum -y install rpmdevtools
+# RHEL/CentOS 8 uses python3.
+if grep -q '^\(CentOS\|Red Hat\)[^0-9]*8\..' /etc/redhat-release; then
+ NAME="python3-google-compute-engine"
+ rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION}
+ sudo yum -y install python36-devel python3-setuptools python36-rpm-macros
+else
+ sudo yum -y install python2-devel python-setuptools python-boto
+fi
+
rm -rf /tmp/rpmpackage
mkdir -p ${rpm_working_dir}/{SOURCES,SPECS}
diff --git a/packages/python-google-compute-engine/setup.py b/packages/python-google-compute-engine/setup.py
index 59a61b3..a91beec 100755
--- a/packages/python-google-compute-engine/setup.py
+++ b/packages/python-google-compute-engine/setup.py
@@ -20,8 +20,10 @@
import setuptools
-install_requires = ['boto', 'setuptools']
-if sys.version_info >= (3, 6):
+install_requires = ['setuptools']
+if sys.version_info < (3, 0):
+ install_requires += ['boto']
+if sys.version_info >= (3, 7):
install_requires += ['distro']
setuptools.setup(
@@ -35,7 +37,7 @@
name='google-compute-engine',
packages=setuptools.find_packages(),
url='https://github.com/GoogleCloudPlatform/compute-image-packages',
- version='2.8.13',
+ version='2.8.15',
# Entry points create scripts in /usr/bin that call a function.
entry_points={
'console_scripts': [
