diff --git a/daisy_workflows/build_deb_packages.sh b/daisy_workflows/build_deb_packages.sh index 7754a7ca..55ee9aec 100755 --- a/daisy_workflows/build_deb_packages.sh +++ b/daisy_workflows/build_deb_packages.sh @@ -33,7 +33,8 @@ fi workdir=$(pwd) mkdir output -sudo apt-get install -y git +apt-get update +apt-get install -y git # Clone the github repo. git clone ${GIT_REPO} -b ${BRANCH} compute-image-packages diff --git a/packages/google-compute-engine/packaging/debian/changelog b/packages/google-compute-engine/packaging/debian/changelog index 81262c93..59b0c074 100644 --- a/packages/google-compute-engine/packaging/debian/changelog +++ b/packages/google-compute-engine/packaging/debian/changelog @@ -1,3 +1,9 @@ +google-compute-engine (1:20190916.00-g2) stable; urgency=medium + + * Update dependencies. + + -- Google Cloud Team Mon, 16 Sep 2019 15:51:24 -0700 + google-compute-engine (1:20190916.00-g1) stable; urgency=medium * Bump version to match python package release. diff --git a/packages/google-compute-engine/packaging/google-compute-engine-el6.spec b/packages/google-compute-engine/packaging/google-compute-engine-el6.spec index ffa5c74c..0e8afb15 100644 --- a/packages/google-compute-engine/packaging/google-compute-engine-el6.spec +++ b/packages/google-compute-engine/packaging/google-compute-engine-el6.spec @@ -15,14 +15,14 @@ Name: google-compute-engine Epoch: 1 Version: %{_version} -Release: g1.el6 +Release: g2.el6 Summary: Google Compute Engine guest environment. License: ASL 2.0 Url: https://github.com/GoogleCloudPlatform/compute-image-packages Source0: %{name}_%{version}.orig.tar.gz Requires: curl Requires: google-compute-engine-oslogin -Requires: python-google-compute-engine = 1:%{version} +Requires: python-google-compute-engine >= 1:20190916.00 Requires: rsyslog # Old packages. Obsoletes: google-compute-engine-init diff --git a/packages/google-compute-engine/packaging/google-compute-engine.spec b/packages/google-compute-engine/packaging/google-compute-engine.spec index 3d0560e3..3f5fcb19 100644 --- a/packages/google-compute-engine/packaging/google-compute-engine.spec +++ b/packages/google-compute-engine/packaging/google-compute-engine.spec @@ -20,7 +20,7 @@ Name: google-compute-engine Epoch: 1 Version: %{_version} -Release: g1%{?dist} +Release: g2%{?dist} Summary: Google Compute Engine guest environment. License: ASL 2.0 Url: https://github.com/GoogleCloudPlatform/compute-image-packages @@ -28,9 +28,9 @@ Source0: %{name}_%{version}.orig.tar.gz Requires: curl Requires: google-compute-engine-oslogin %if 0%{?rhel} == 8 -Requires: python3-google-compute-engine = 1:%{version} +Requires: python3-google-compute-engine >= 1:20190916.00 %else -Requires: python-google-compute-engine = 1:%{version} +Requires: python-google-compute-engine >= 1:20190916.00 %endif Requires: rsyslog diff --git a/packages/python-google-compute-engine/google_compute_engine/distro_lib/helpers.py b/packages/python-google-compute-engine/google_compute_engine/distro_lib/helpers.py index a0be6a03..c20b2419 100644 --- a/packages/python-google-compute-engine/google_compute_engine/distro_lib/helpers.py +++ b/packages/python-google-compute-engine/google_compute_engine/distro_lib/helpers.py @@ -17,6 +17,7 @@ import os import subprocess +import time def CallDhclient( @@ -68,6 +69,23 @@ def CallDhclientIpv6(interfaces, logger, dhclient_script=None, interfaces) return + # Check for a 'tentative' IPv6 address which would prevent `dhclient -6` from + # succeeding below. This should only take 1 second, but we try for up to 5. + command = ['ip', '-6', '-o', 'a', 's', 'dev', interfaces[0], 'scope', + 'link', 'tentative'] + for i in range(5): + output = '' + try: + output = subprocess.check_output(command) + except subprocess.CalledProcessError as e: + logger.warning('Could not confirm tentative IPv6 address: %s.', e.output) + if output: + logger.info('Found tentative ipv6 link address %s, sleeping 1 second.', + output.strip()) + time.sleep(1) + else: + break + if dhclient_script and os.path.exists(dhclient_script): dhclient_command += ['-sf', dhclient_script] diff --git a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py index cb1a2a6d..dbd3d8f0 100755 --- a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py +++ b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py @@ -28,7 +28,7 @@ from google_compute_engine import logger from google_compute_engine import metadata_watcher from google_compute_engine.boto import boto_config -from google_compute_engine.compat import urlerror +from google_compute_engine.compat import distro_name, urlerror from google_compute_engine.compat import urlrequest from google_compute_engine.instance_setup import instance_config @@ -65,17 +65,27 @@ def __init__(self, debug=False): instance_config_metadata = self._GetInstanceConfig() self.instance_config = instance_config.InstanceConfig( logger=self.logger, instance_config_metadata=instance_config_metadata) + if self.instance_config.GetOptionBool('InstanceSetup', 'set_host_keys'): host_key_types = self.instance_config.GetOptionString( 'InstanceSetup', 'host_key_types') self._SetSshHostKeys(host_key_types=host_key_types) + if self.instance_config.GetOptionBool('InstanceSetup', 'set_boto_config'): self._SetupBotoConfig() + + # machineType is e.g. u'projects/00000000000000/machineTypes/n1-standard-1' + machineType = self.metadata_dict['instance']['machineType'].split('/')[-1] + if machineType.startswith("e2-") and 'bsd' not in distro_name: # Not yet supported on BSD. + subprocess.call(["sysctl", "vm.overcommit_memory=1"]) + if self.instance_config.GetOptionBool( 'InstanceSetup', 'optimize_local_ssd'): self._RunScript('google_optimize_local_ssd') + if self.instance_config.GetOptionBool('InstanceSetup', 'set_multiqueue'): self._RunScript('google_set_multiqueue') + try: self.instance_config.WriteConfig() except (IOError, OSError) as e: @@ -205,10 +215,9 @@ def _SetSshHostKeys(self, host_key_types=None): Args: host_key_types: string, a comma separated list of host key types. """ - section = 'Instance' instance_id = self._GetInstanceId() if instance_id != self.instance_config.GetOptionString( - section, 'instance_id'): + 'instance', 'instance_id'): self.logger.info('Generating SSH host keys for instance %s.', instance_id) file_regex = re.compile(r'ssh_host_(?P[a-z0-9]*)_key\Z') key_dir = '/etc/ssh' @@ -222,7 +231,7 @@ def _SetSshHostKeys(self, host_key_types=None): if key_data: self._WriteHostKeyToGuestAttributes(key_data[0], key_data[1]) self._StartSshd() - self.instance_config.SetOption(section, 'instance_id', str(instance_id)) + self.instance_config.SetOption('Instance', 'instance_id', str(instance_id)) def _GetNumericProjectId(self): """Get the numeric project ID. diff --git a/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py b/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py index a4182199..e255f86a 100644 --- a/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py +++ b/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py @@ -46,7 +46,7 @@ def testInstanceSetup(self, mock_logger, mock_watcher, mock_config): mock_logger_instance = mock.Mock() mock_logger.Logger.return_value = mock_logger_instance mock_watcher_instance = mock.Mock() - mock_watcher_instance.GetMetadata.return_value = {'hello': 'world'} + mock_watcher_instance.GetMetadata.return_value = {'hello': 'world', 'instance': {'machineType': 'fake'}} mock_watcher.MetadataWatcher.return_value = mock_watcher_instance mock_config_instance = mock.Mock() mock_config_instance.GetOptionBool.return_value = True @@ -91,7 +91,7 @@ def testInstanceSetup(self, mock_logger, mock_watcher, mock_config): mock.call.config.InstanceConfig().WriteConfig(), ] self.assertEqual(mocks.mock_calls, expected_calls) - self.assertEqual(mock_setup.metadata_dict, {'hello': 'world'}) + self.assertEqual(mock_setup.metadata_dict, {'hello': 'world', 'instance': {'machineType': 'fake'}}) @mock.patch('google_compute_engine.instance_setup.instance_setup.instance_config') @mock.patch('google_compute_engine.instance_setup.instance_setup.metadata_watcher') @@ -386,7 +386,7 @@ def testSetSshHostKeysFirstBoot(self, mock_listdir): ] self.assertEqual(sorted(mock_generate_key.mock_calls), expected_calls) - self.mock_instance_config.SetOption.assert_called_once_with( + self.mock_instance_config.SetOption.assert_called_with( 'Instance', 'instance_id', '123') def testGetNumericProjectId(self): diff --git a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py index 3de425b6..fd07423f 100644 --- a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py +++ b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py @@ -97,7 +97,7 @@ def _DownloadAuthUrl(self, url, dest_dir): if not self.token: response = self.watcher.GetMetadata( - self.token_metadata_key, recursive=False, retry=False) + self.token_metadata_key, recursive=False, retries=1) if not response: self.logger.info( diff --git a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py index 0df8830f..4019bbed 100644 --- a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py +++ b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py @@ -89,7 +89,7 @@ def testDownloadAuthUrlExceptionAndToken( # GetMetadata includes a prefix, so remove it. stripped_url = token_url.replace(metadata_prefix, '') mock_get_metadata.assert_called_once_with( - stripped_url, recursive=False, retry=False) + stripped_url, recursive=False, retries=1) self.assertEqual(self.retriever.token, 'foo bar') @@ -119,7 +119,7 @@ def testDownloadAuthUrlFallback( prefix = 'http://metadata.google.internal/computeMetadata/v1/' stripped_url = token_url.replace(prefix, '') mock_get_metadata.assert_called_once_with( - stripped_url, recursive=False, retry=False) + stripped_url, recursive=False, retries=1) mock_download_url.assert_called_once_with(auth_url, self.dest_dir) self.assertIsNone(self.retriever.token) diff --git a/packages/python-google-compute-engine/google_compute_engine/metadata_watcher.py b/packages/python-google-compute-engine/google_compute_engine/metadata_watcher.py index 72a12f97..bf688cfa 100644 --- a/packages/python-google-compute-engine/google_compute_engine/metadata_watcher.py +++ b/packages/python-google-compute-engine/google_compute_engine/metadata_watcher.py @@ -154,7 +154,7 @@ def _GetMetadataUpdate( def _HandleMetadataUpdate( self, metadata_key='', recursive=True, wait=True, timeout=None, - retry=True): + retries=5): """Wait for a successful metadata response. Args: @@ -162,25 +162,23 @@ def _HandleMetadataUpdate( recursive: bool, True if we should recursively watch for metadata changes. wait: bool, True if we should wait for a metadata change. timeout: int, timeout in seconds for returning metadata output. - retry: bool, True if we should retry on failure. + retries: int, Number of times to retry obtaining metadata. Returns: json, the deserialized contents of the metadata server. """ exception = None - while True: + while retries >= 0: try: return self._GetMetadataUpdate( metadata_key=metadata_key, recursive=recursive, wait=wait, timeout=timeout) except (httpclient.HTTPException, socket.error, urlerror.URLError) as e: + retries -= 1 if not isinstance(e, type(exception)): exception = e self.logger.error('GET request error retrieving metadata. %s.', e) - if retry: - continue - else: - break + time.sleep(1) def WatchMetadata( self, handler, metadata_key='', recursive=True, timeout=None): @@ -202,7 +200,7 @@ def WatchMetadata( self.logger.exception('Exception calling the response handler. %s.', e) def GetMetadata( - self, metadata_key='', recursive=True, timeout=None, retry=True): + self, metadata_key='', recursive=True, timeout=None, retries=5): """Retrieve the contents of metadata server for a metadata key. Args: @@ -216,4 +214,4 @@ def GetMetadata( """ return self._HandleMetadataUpdate( metadata_key=metadata_key, recursive=recursive, wait=False, - timeout=timeout, retry=retry) + timeout=timeout, retries=retries) diff --git a/packages/python-google-compute-engine/google_compute_engine/tests/metadata_watcher_test.py b/packages/python-google-compute-engine/google_compute_engine/tests/metadata_watcher_test.py index 1bce5091..9c0ca464 100644 --- a/packages/python-google-compute-engine/google_compute_engine/tests/metadata_watcher_test.py +++ b/packages/python-google-compute-engine/google_compute_engine/tests/metadata_watcher_test.py @@ -259,12 +259,12 @@ def testHandleMetadataUpdateException(self): metadata_key = 'instance/id' recursive = False wait = False - retry = True + retries = 5 self.assertEqual( self.mock_watcher._HandleMetadataUpdate( metadata_key=metadata_key, recursive=recursive, wait=wait, - timeout=None, retry=retry), + timeout=None, retries=retries), {}) expected_calls = [ mock.call( @@ -282,12 +282,12 @@ def testHandleMetadataUpdateExceptionNoRetry(self): metadata_key = 'instance/id' recursive = False wait = False - retry = False + retries = 0 self.assertIsNone( self.mock_watcher._HandleMetadataUpdate( metadata_key=metadata_key, recursive=recursive, wait=wait, - timeout=None, retry=retry)) + timeout=None, retries=retries)) expected_calls = [ mock.call( metadata_key=metadata_key, recursive=recursive, wait=wait, @@ -333,7 +333,7 @@ def testGetMetadata(self): self.assertEqual(self.mock_watcher.GetMetadata(), {}) mock_response.assert_called_once_with( - metadata_key='', recursive=True, wait=False, timeout=None, retry=True) + metadata_key='', recursive=True, wait=False, timeout=None, retries=5) self.mock_watcher.logger.exception.assert_not_called() def testGetMetadataArgs(self): @@ -342,15 +342,15 @@ def testGetMetadataArgs(self): self.mock_watcher._HandleMetadataUpdate = mock_response metadata_key = 'instance/id' recursive = False - retry = False + retries = 5 response = self.mock_watcher.GetMetadata( metadata_key=metadata_key, recursive=recursive, timeout=60, - retry=retry) + retries=retries) self.assertEqual(response, {}) mock_response.assert_called_once_with( metadata_key=metadata_key, recursive=False, wait=False, timeout=60, - retry=False) + retries=5) self.mock_watcher.logger.exception.assert_not_called() diff --git a/packages/python-google-compute-engine/packaging/debian/changelog b/packages/python-google-compute-engine/packaging/debian/changelog index edf3f0c1..3a797216 100644 --- a/packages/python-google-compute-engine/packaging/debian/changelog +++ b/packages/python-google-compute-engine/packaging/debian/changelog @@ -1,3 +1,11 @@ +python-google-compute-engine (1:20191112.00-g1) stable; urgency=medium + + * Enable sysctl change for E2 platform. + * Retry metadata lookups in agent. + * Wait for link-local IPv6 addresses to resolve. + + -- Google Cloud Team Mon, 16 Sep 2019 15:52:24 -0700 + python-google-compute-engine (1:20190916.00-g1) stable; urgency=medium * Revert VM rename feature. diff --git a/packages/python-google-compute-engine/packaging/setup_deb.sh b/packages/python-google-compute-engine/packaging/setup_deb.sh index 1b64ba76..fc4feac6 100755 --- a/packages/python-google-compute-engine/packaging/setup_deb.sh +++ b/packages/python-google-compute-engine/packaging/setup_deb.sh @@ -14,7 +14,7 @@ # limitations under the License. NAME="python-google-compute-engine" -VERSION="20190916.00" +VERSION="20191112.00" working_dir=${PWD} if [[ $(basename "$working_dir") != $NAME ]]; then diff --git a/packages/python-google-compute-engine/packaging/setup_rpm.sh b/packages/python-google-compute-engine/packaging/setup_rpm.sh index acd617c9..d3c0d223 100755 --- a/packages/python-google-compute-engine/packaging/setup_rpm.sh +++ b/packages/python-google-compute-engine/packaging/setup_rpm.sh @@ -14,7 +14,7 @@ # limitations under the License. NAME="python-google-compute-engine" -VERSION="20190916.00" +VERSION="20191112.00" rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION} working_dir=${PWD} diff --git a/packages/python-google-compute-engine/setup.py b/packages/python-google-compute-engine/setup.py index c8900f93..e248ca84 100755 --- a/packages/python-google-compute-engine/setup.py +++ b/packages/python-google-compute-engine/setup.py @@ -37,7 +37,7 @@ name='google-compute-engine', packages=setuptools.find_packages(), url='https://github.com/GoogleCloudPlatform/compute-image-packages', - version='20190916.0', + version='20191112.0', # Entry points create scripts in /usr/bin that call a function. entry_points={ 'console_scripts': [