Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Merged cloud test refactor from 2019.2.1 #54326

Merged
merged 6 commits into from
Aug 30, 2019
Merged

Merged cloud test refactor from 2019.2.1 #54326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
802702e
Merged cloud test refactor from 2019.2.1
Akm0d Aug 27, 2019
472578f
Merged cloud changes from 2019.2.1
Akm0d Aug 28, 2019
8482e03
merged cloud utils from 2019.2.1
Akm0d Aug 28, 2019
e5d3baa
fixed docbuildstrings
Akm0d Aug 29, 2019
90d4d7b
More doc fixes
Akm0d Aug 29, 2019
52739cc
merged docstrings from neon
Akm0d Aug 29, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
214 changes: 96 additions & 118 deletions salt/cloud/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,6 @@ def _call(queue, args, kwargs):
queue.put('ERROR')
queue.put('Exception')
queue.put('{0}\n{1}\n'.format(ex, trace))
except SystemExit as ex:
trace = traceback.format_exc()
queue.put('ERROR')
queue.put('System exit')
queue.put('{0}\n{1}\n'.format(ex, trace))
return ret
return _call

Expand Down Expand Up @@ -189,10 +184,8 @@ class CloudClient(object):
def __init__(self, path=None, opts=None, config_dir=None, pillars=None):
if opts:
self.opts = opts
elif path:
self.opts = salt.config.cloud_config(path)
else:
self.opts = salt.config.cloud_config()
self.opts = salt.config.cloud_config(path)

# Check the cache-dir exists. If not, create it.
v_dirs = [self.opts['cachedir']]
Expand Down Expand Up @@ -524,7 +517,6 @@ class Cloud(object):
'''
def __init__(self, opts):
self.opts = opts
self.client = CloudClient(opts=self.opts)
self.clouds = salt.loader.clouds(self.opts)
self.__filter_non_working_providers()
self.__cached_provider_queries = {}
Expand Down Expand Up @@ -2058,138 +2050,124 @@ def run_map(self, dmap):

# Now sort the create list based on dependencies
create_list = sorted(six.iteritems(dmap['create']), key=lambda x: x[1]['level'])
full_map = dmap['create'].copy()
if 'existing' in dmap:
full_map.update(dmap['existing'])
possible_master_list = sorted(six.iteritems(full_map), key=lambda x: x[1]['level'])
output = {}
if self.opts['parallel']:
parallel_data = []
master_name = None
master_minion_name = None
master_host = None
master_finger = None
for name, profile in possible_master_list:
if profile.get('make_master', False) is True:
master_name = name
master_profile = profile

if master_name:
# If the master already exists, get the host
if master_name not in dmap['create']:
master_host = self.client.query()
for provider_part in master_profile['provider'].split(':'):
master_host = master_host[provider_part]
master_host = master_host[master_name][master_profile.get('ssh_interface', 'public_ips')]
if not master_host:
raise SaltCloudSystemExit(
'Could not get the hostname of master {}.'.format(master_name)
)
# Otherwise, deploy it as a new master
else:
master_minion_name = master_name
log.debug('Creating new master \'%s\'', master_name)
if salt.config.get_cloud_config_value(
'deploy',
try:
master_name, master_profile = next((
(name, profile) for name, profile in create_list
if profile.get('make_master', False) is True
))
master_minion_name = master_name
log.debug('Creating new master \'%s\'', master_name)
if salt.config.get_cloud_config_value(
'deploy',
master_profile,
self.opts
) is False:
raise SaltCloudSystemExit(
'Cannot proceed with \'make_master\' when salt deployment '
'is disabled(ex: --no-deploy).'
)

# Generate the master keys
log.debug('Generating master keys for \'%s\'', master_profile['name'])
priv, pub = salt.utils.cloud.gen_keys(
salt.config.get_cloud_config_value(
'keysize',
master_profile,
self.opts
) is False:
raise SaltCloudSystemExit(
'Cannot proceed with \'make_master\' when salt deployment '
'is disabled(ex: --no-deploy).'
)
)
)
master_profile['master_pub'] = pub
master_profile['master_pem'] = priv

# Generate the master keys
log.debug('Generating master keys for \'%s\'', master_profile['name'])
# Generate the fingerprint of the master pubkey in order to
# mitigate man-in-the-middle attacks
master_temp_pub = salt.utils.files.mkstemp()
with salt.utils.files.fopen(master_temp_pub, 'w') as mtp:
mtp.write(pub)
master_finger = salt.utils.crypt.pem_finger(master_temp_pub, sum_type=self.opts['hash_type'])
os.unlink(master_temp_pub)

if master_profile.get('make_minion', True) is True:
master_profile.setdefault('minion', {})
if 'id' in master_profile['minion']:
master_minion_name = master_profile['minion']['id']
# Set this minion's master as local if the user has not set it
if 'master' not in master_profile['minion']:
master_profile['minion']['master'] = '127.0.0.1'
if master_finger is not None:
master_profile['master_finger'] = master_finger

# Generate the minion keys to pre-seed the master:
for name, profile in create_list:
make_minion = salt.config.get_cloud_config_value(
'make_minion', profile, self.opts, default=True
)
if make_minion is False:
continue

log.debug('Generating minion keys for \'%s\'', profile['name'])
priv, pub = salt.utils.cloud.gen_keys(
salt.config.get_cloud_config_value(
'keysize',
master_profile,
profile,
self.opts
)
)
master_profile['master_pub'] = pub
master_profile['master_pem'] = priv

# Generate the fingerprint of the master pubkey in order to
# mitigate man-in-the-middle attacks
master_temp_pub = salt.utils.files.mkstemp()
with salt.utils.files.fopen(master_temp_pub, 'w') as mtp:
mtp.write(pub)
master_finger = salt.utils.crypt.pem_finger(master_temp_pub, sum_type=self.opts['hash_type'])
os.unlink(master_temp_pub)

if master_profile.get('make_minion', True) is True:
master_profile.setdefault('minion', {})
if 'id' in master_profile['minion']:
master_minion_name = master_profile['minion']['id']
# Set this minion's master as local if the user has not set it
if 'master' not in master_profile['minion']:
master_profile['minion']['master'] = '127.0.0.1'
if master_finger is not None:
master_profile['master_finger'] = master_finger

# Generate the minion keys to pre-seed the master:
for name, profile in create_list:
make_minion = salt.config.get_cloud_config_value(
'make_minion', profile, self.opts, default=True
)
if make_minion is False:
continue
profile['pub_key'] = pub
profile['priv_key'] = priv
# Store the minion's public key in order to be pre-seeded in
# the master
master_profile.setdefault('preseed_minion_keys', {})
master_profile['preseed_minion_keys'].update({name: pub})

local_master = False
if master_profile['minion'].get('local_master', False) and \
master_profile['minion'].get('master', None) is not None:
# The minion is explicitly defining a master and it's
# explicitly saying it's the local one
local_master = True

log.debug('Generating minion keys for \'%s\'', profile['name'])
priv, pub = salt.utils.cloud.gen_keys(
salt.config.get_cloud_config_value(
'keysize',
profile,
self.opts
)
)
profile['pub_key'] = pub
profile['priv_key'] = priv
# Store the minion's public key in order to be pre-seeded in
# the master
master_profile.setdefault('preseed_minion_keys', {})
master_profile['preseed_minion_keys'].update({name: pub})

local_master = False
if master_profile['minion'].get('local_master', False) and \
master_profile['minion'].get('master', None) is not None:
# The minion is explicitly defining a master and it's
# explicitly saying it's the local one
local_master = True

out = self.create(master_profile, local_master=local_master)

if not isinstance(out, dict):
log.debug('Master creation details is not a dictionary: %s', out)

elif 'Errors' in out:
raise SaltCloudSystemExit(
'An error occurred while creating the master, not '
'continuing: {0}'.format(out['Errors'])
out = self.create(master_profile, local_master=local_master)

if not isinstance(out, dict):
log.debug(
'Master creation details is not a dictionary: {0}'.format(
out
)
)

deploy_kwargs = (
self.opts.get('show_deploy_args', False) is True and
# Get the needed data
out.get('deploy_kwargs', {}) or
# Strip the deploy_kwargs from the returned data since we don't
# want it shown in the console.
out.pop('deploy_kwargs', {})
elif 'Errors' in out:
raise SaltCloudSystemExit(
'An error occurred while creating the master, not '
'continuing: {0}'.format(out['Errors'])
)

master_host = deploy_kwargs.get('salt_host', deploy_kwargs.get('host', None))
if master_host is None:
raise SaltCloudSystemExit(
'Host for new master {0} was not found, '
'aborting map'.format(
master_name
)
deploy_kwargs = (
self.opts.get('show_deploy_args', False) is True and
# Get the needed data
out.get('deploy_kwargs', {}) or
# Strip the deploy_kwargs from the returned data since we don't
# want it shown in the console.
out.pop('deploy_kwargs', {})
)

master_host = deploy_kwargs.get('salt_host', deploy_kwargs.get('host', None))
if master_host is None:
raise SaltCloudSystemExit(
'Host for new master {0} was not found, '
'aborting map'.format(
master_name
)
output[master_name] = out
else:
)
output[master_name] = out
except StopIteration:
log.debug('No make_master found in map')
# Local master?
# Generate the fingerprint of the master pubkey in order to
Expand Down
8 changes: 3 additions & 5 deletions salt/cloud/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import salt.utils.parsers
import salt.utils.user
from salt.exceptions import SaltCloudException, SaltCloudSystemExit
from salt.utils.verify import check_user, verify_env, verify_files, verify_log
from salt.utils.verify import check_user, verify_env, verify_log_files, verify_log

# Import 3rd-party libs
from salt.ext import six
Expand Down Expand Up @@ -70,11 +70,9 @@ def run(self):
root_dir=self.config['root_dir'],
)
logfile = self.config['log_file']
if logfile is not None and not logfile.startswith('tcp://') \
and not logfile.startswith('udp://') \
and not logfile.startswith('file://'):
if logfile is not None:
# Logfile is not using Syslog, verify
verify_files([logfile], salt_master_user)
verify_log_files([logfile], salt_master_user)
except (IOError, OSError) as err:
log.error('Error while verifying the environment: %s', err)
sys.exit(err.errno)
Expand Down
48 changes: 18 additions & 30 deletions salt/cloud/clouds/azurearm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,6 @@
* ``client_id``
* ``secret``

if using MSI-style authentication:
* ``subscription_id``

Optional provider parameters:

**cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. Possible values:
Expand Down Expand Up @@ -254,22 +251,14 @@ def __is_provider_configured(opts, provider, required_keys=()):
provider = __is_provider_configured(
__opts__,
__active_provider_name__ or __virtualname__,
('subscription_id', 'tenant', 'client_id', 'secret'),
('subscription_id', 'tenant', 'client_id', 'secret')
)

if provider is False:
provider = __is_provider_configured(
__opts__,
__active_provider_name__ or __virtualname__,
('subscription_id', 'username', 'password'),
)

if provider is False:
# check if using MSI style credentials...
provider = config.is_provider_configured(
__opts__,
__active_provider_name__ or __virtualname__,
required_keys=('subscription_id',),
('subscription_id', 'username', 'password')
)

return provider
Expand Down Expand Up @@ -322,13 +311,11 @@ def get_conn(client_type):
)
conn_kwargs.update({'client_id': client_id, 'secret': secret,
'tenant': tenant})

username = config.get_cloud_config_value(
'username',
get_configured_provider(), __opts__, search_global=False
)

if username is not None:
else:
username = config.get_cloud_config_value(
'username',
get_configured_provider(), __opts__, search_global=False
)
password = config.get_cloud_config_value(
'password',
get_configured_provider(), __opts__, search_global=False
Expand Down Expand Up @@ -1048,15 +1035,6 @@ def request_instance(vm_):
default=False
)

vm_password = salt.utils.stringutils.to_str(
config.get_cloud_config_value(
'ssh_password', vm_, __opts__, search_global=True,
default=config.get_cloud_config_value(
'win_password', vm_, __opts__, search_global=True
)
)
)

os_kwargs = {}
win_installer = config.get_cloud_config_value(
'win_installer', vm_, __opts__, search_global=True
Expand All @@ -1074,6 +1052,16 @@ def request_instance(vm_):
ssh=sshconfiguration,
)
os_kwargs['linux_configuration'] = linuxconfiguration
vm_password = None
else:
vm_password = salt.utils.stringutils.to_str(
config.get_cloud_config_value(
'ssh_password', vm_, __opts__, search_global=True,
default=config.get_cloud_config_value(
'win_password', vm_, __opts__, search_global=True
)
)
)

if win_installer or (vm_password is not None and not disable_password_authentication):
if not isinstance(vm_password, str):
Expand Down Expand Up @@ -1340,7 +1328,7 @@ def request_instance(vm_):
),
network_profile=NetworkProfile(
network_interfaces=[
NetworkInterfaceReference(vm_['iface_id']),
NetworkInterfaceReference(id=vm_['iface_id']),
],
),
availability_set=availability_set,
Expand Down
Loading